{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.5578800557880056,
  "eval_steps": 300,
  "global_step": 250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0022315202231520223,
      "grad_norm": 8.175573348999023,
      "learning_rate": 4.444444444444445e-06,
      "logits/chosen": -0.48349103331565857,
      "logits/rejected": -0.652603030204773,
      "logps/chosen": -8.859519004821777,
      "logps/rejected": -16.016435623168945,
      "loss": 1.3468515872955322,
      "memory(GiB)": 33.44,
      "nll_loss": 0.6537042856216431,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.010093
    },
    {
      "epoch": 0.004463040446304045,
      "grad_norm": 6.496051788330078,
      "learning_rate": 8.88888888888889e-06,
      "logits/chosen": -0.5174899101257324,
      "logits/rejected": -0.6262180805206299,
      "logps/chosen": -9.287378311157227,
      "logps/rejected": -13.981985092163086,
      "loss": 1.290852665901184,
      "memory(GiB)": 36.3,
      "nll_loss": 0.5977055430412292,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.010436
    },
    {
      "epoch": 0.0066945606694560665,
      "grad_norm": 5.3426923751831055,
      "learning_rate": 1.3333333333333333e-05,
      "logits/chosen": -0.4254978895187378,
      "logits/rejected": -0.6118676662445068,
      "logps/chosen": -6.5775346755981445,
      "logps/rejected": -16.196733474731445,
      "loss": 1.1887776851654053,
      "memory(GiB)": 36.3,
      "nll_loss": 0.49580031633377075,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.0025197849608957767,
      "rewards/margins": 0.0005687186494469643,
      "rewards/rejected": -0.0030885031446814537,
      "step": 3,
      "train_speed(iter/s)": 0.010535
    },
    {
      "epoch": 0.00892608089260809,
      "grad_norm": 5.717321395874023,
      "learning_rate": 1.777777777777778e-05,
      "logits/chosen": -0.5246356129646301,
      "logits/rejected": -0.651706337928772,
      "logps/chosen": -7.447962760925293,
      "logps/rejected": -16.996183395385742,
      "loss": 1.3215211629867554,
      "memory(GiB)": 39.38,
      "nll_loss": 0.632453203201294,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.004622042179107666,
      "rewards/margins": 0.008373789489269257,
      "rewards/rejected": -0.0037517473101615906,
      "step": 4,
      "train_speed(iter/s)": 0.010586
    },
    {
      "epoch": 0.011157601115760111,
      "grad_norm": 7.188594341278076,
      "learning_rate": 2.2222222222222223e-05,
      "logits/chosen": -0.5062788128852844,
      "logits/rejected": -0.6033568382263184,
      "logps/chosen": -7.093954086303711,
      "logps/rejected": -15.186193466186523,
      "loss": 1.0765105485916138,
      "memory(GiB)": 39.38,
      "nll_loss": 0.4042052626609802,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.018988817930221558,
      "rewards/margins": 0.04282933473587036,
      "rewards/rejected": -0.023840520530939102,
      "step": 5,
      "train_speed(iter/s)": 0.010619
    },
    {
      "epoch": 0.013389121338912133,
      "grad_norm": 3.8232157230377197,
      "learning_rate": 2.6666666666666667e-05,
      "logits/chosen": -0.4826778173446655,
      "logits/rejected": -0.5863982439041138,
      "logps/chosen": -6.322361946105957,
      "logps/rejected": -16.8499755859375,
      "loss": 1.2770025730133057,
      "memory(GiB)": 39.38,
      "nll_loss": 0.6041065454483032,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.007987724617123604,
      "rewards/margins": 0.0449829027056694,
      "rewards/rejected": -0.03699517622590065,
      "step": 6,
      "train_speed(iter/s)": 0.010645
    },
    {
      "epoch": 0.015620641562064157,
      "grad_norm": 3.0141377449035645,
      "learning_rate": 3.111111111111111e-05,
      "logits/chosen": -0.557126522064209,
      "logits/rejected": -0.6060188412666321,
      "logps/chosen": -5.420467853546143,
      "logps/rejected": -11.815977096557617,
      "loss": 1.1219334602355957,
      "memory(GiB)": 39.38,
      "nll_loss": 0.46769219636917114,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.010057949461042881,
      "rewards/margins": 0.10709290951490402,
      "rewards/rejected": -0.09703496098518372,
      "step": 7,
      "train_speed(iter/s)": 0.010666
    },
    {
      "epoch": 0.01785216178521618,
      "grad_norm": 2.8756191730499268,
      "learning_rate": 3.555555555555556e-05,
      "logits/chosen": -0.5616711378097534,
      "logits/rejected": -0.6365323066711426,
      "logps/chosen": -6.283439636230469,
      "logps/rejected": -19.981098175048828,
      "loss": 0.8637306690216064,
      "memory(GiB)": 39.38,
      "nll_loss": 0.33207178115844727,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.011994147673249245,
      "rewards/margins": 0.4002670347690582,
      "rewards/rejected": -0.4122611880302429,
      "step": 8,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.0200836820083682,
      "grad_norm": 5.291012763977051,
      "learning_rate": 4e-05,
      "logits/chosen": -0.502879798412323,
      "logits/rejected": -0.5977268815040588,
      "logps/chosen": -16.167051315307617,
      "logps/rejected": -20.485633850097656,
      "loss": 1.618671178817749,
      "memory(GiB)": 39.38,
      "nll_loss": 1.0418576002120972,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.21173161268234253,
      "rewards/margins": 0.39757153391838074,
      "rewards/rejected": -0.6093031167984009,
      "step": 9,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.022315202231520222,
      "grad_norm": 5.504859447479248,
      "learning_rate": 4.4444444444444447e-05,
      "logits/chosen": -0.4140257239341736,
      "logits/rejected": -0.5436943173408508,
      "logps/chosen": -10.317049026489258,
      "logps/rejected": -26.558115005493164,
      "loss": 1.271435260772705,
      "memory(GiB)": 39.38,
      "nll_loss": 0.6719653606414795,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2977833151817322,
      "rewards/margins": 0.4953876733779907,
      "rewards/rejected": -0.7931710481643677,
      "step": 10,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.024546722454672244,
      "grad_norm": 8.8348970413208,
      "learning_rate": 4.888888888888889e-05,
      "logits/chosen": -0.4538601040840149,
      "logits/rejected": -0.5884804129600525,
      "logps/chosen": -15.600846290588379,
      "logps/rejected": -27.856977462768555,
      "loss": 1.6294305324554443,
      "memory(GiB)": 39.38,
      "nll_loss": 1.0609474182128906,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.48508310317993164,
      "rewards/margins": 0.41195282340049744,
      "rewards/rejected": -0.8970359563827515,
      "step": 11,
      "train_speed(iter/s)": 0.0107
    },
    {
      "epoch": 0.026778242677824266,
      "grad_norm": 4.871168613433838,
      "learning_rate": 5.333333333333333e-05,
      "logits/chosen": -0.4462035596370697,
      "logits/rejected": -0.5622009634971619,
      "logps/chosen": -10.827975273132324,
      "logps/rejected": -27.8737850189209,
      "loss": 1.309844732284546,
      "memory(GiB)": 42.78,
      "nll_loss": 0.6817098259925842,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.34655484557151794,
      "rewards/margins": 0.38605934381484985,
      "rewards/rejected": -0.7326142191886902,
      "step": 12,
      "train_speed(iter/s)": 0.010701
    },
    {
      "epoch": 0.02900976290097629,
      "grad_norm": 5.154696464538574,
      "learning_rate": 5.7777777777777776e-05,
      "logits/chosen": -0.45840126276016235,
      "logits/rejected": -0.5749193429946899,
      "logps/chosen": -9.034074783325195,
      "logps/rejected": -23.558507919311523,
      "loss": 1.3584285974502563,
      "memory(GiB)": 42.78,
      "nll_loss": 0.8017209768295288,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.15672500431537628,
      "rewards/margins": 0.44559869170188904,
      "rewards/rejected": -0.6023237109184265,
      "step": 13,
      "train_speed(iter/s)": 0.010704
    },
    {
      "epoch": 0.031241283124128313,
      "grad_norm": 2.928802013397217,
      "learning_rate": 6.222222222222222e-05,
      "logits/chosen": -0.4339243173599243,
      "logits/rejected": -0.5222603678703308,
      "logps/chosen": -7.9224724769592285,
      "logps/rejected": -25.263307571411133,
      "loss": 1.176775574684143,
      "memory(GiB)": 46.23,
      "nll_loss": 0.548651397228241,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09932564944028854,
      "rewards/margins": 0.27659934759140015,
      "rewards/rejected": -0.3759249448776245,
      "step": 14,
      "train_speed(iter/s)": 0.010698
    },
    {
      "epoch": 0.03347280334728033,
      "grad_norm": 1.9422496557235718,
      "learning_rate": 6.666666666666667e-05,
      "logits/chosen": -0.4753529131412506,
      "logits/rejected": -0.5179453492164612,
      "logps/chosen": -8.660318374633789,
      "logps/rejected": -16.743135452270508,
      "loss": 1.1362940073013306,
      "memory(GiB)": 46.23,
      "nll_loss": 0.5188332796096802,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09274716675281525,
      "rewards/margins": 0.20326045155525208,
      "rewards/rejected": -0.29600760340690613,
      "step": 15,
      "train_speed(iter/s)": 0.010702
    },
    {
      "epoch": 0.03570432357043236,
      "grad_norm": 1.7832196950912476,
      "learning_rate": 7.111111111111112e-05,
      "logits/chosen": -0.39417293667793274,
      "logits/rejected": -0.5316826701164246,
      "logps/chosen": -4.724964618682861,
      "logps/rejected": -19.69357681274414,
      "loss": 1.0538526773452759,
      "memory(GiB)": 46.23,
      "nll_loss": 0.4770840108394623,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06691457331180573,
      "rewards/margins": 0.27725517749786377,
      "rewards/rejected": -0.21034058928489685,
      "step": 16,
      "train_speed(iter/s)": 0.010702
    },
    {
      "epoch": 0.03793584379358438,
      "grad_norm": 1.5563856363296509,
      "learning_rate": 7.555555555555556e-05,
      "logits/chosen": -0.4563882350921631,
      "logits/rejected": -0.5541551113128662,
      "logps/chosen": -9.05321979522705,
      "logps/rejected": -18.82474708557129,
      "loss": 1.2152374982833862,
      "memory(GiB)": 46.23,
      "nll_loss": 0.6201823353767395,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.11768844723701477,
      "rewards/margins": 0.2521967887878418,
      "rewards/rejected": -0.13450834155082703,
      "step": 17,
      "train_speed(iter/s)": 0.010706
    },
    {
      "epoch": 0.0401673640167364,
      "grad_norm": 1.9268314838409424,
      "learning_rate": 8e-05,
      "logits/chosen": -0.4476441740989685,
      "logits/rejected": -0.4938126504421234,
      "logps/chosen": -9.981496810913086,
      "logps/rejected": -14.752214431762695,
      "loss": 1.1654764413833618,
      "memory(GiB)": 46.23,
      "nll_loss": 0.46348586678504944,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.008895257487893105,
      "rewards/margins": 0.039339594542980194,
      "rewards/rejected": -0.030444344505667686,
      "step": 18,
      "train_speed(iter/s)": 0.010707
    },
    {
      "epoch": 0.042398884239888426,
      "grad_norm": 1.6981456279754639,
      "learning_rate": 8.444444444444444e-05,
      "logits/chosen": -0.49693721532821655,
      "logits/rejected": -0.5383259057998657,
      "logps/chosen": -7.632569313049316,
      "logps/rejected": -17.26980209350586,
      "loss": 1.1578692197799683,
      "memory(GiB)": 46.23,
      "nll_loss": 0.5097256898880005,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.06525340676307678,
      "rewards/margins": 0.123507060110569,
      "rewards/rejected": -0.05825363099575043,
      "step": 19,
      "train_speed(iter/s)": 0.010709
    },
    {
      "epoch": 0.044630404463040445,
      "grad_norm": 2.013322591781616,
      "learning_rate": 8.888888888888889e-05,
      "logits/chosen": -0.4885023236274719,
      "logits/rejected": -0.5652343034744263,
      "logps/chosen": -6.508520603179932,
      "logps/rejected": -13.635923385620117,
      "loss": 1.0892918109893799,
      "memory(GiB)": 46.23,
      "nll_loss": 0.45972177386283875,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.06720655411481857,
      "rewards/margins": 0.18712735176086426,
      "rewards/rejected": -0.11992079019546509,
      "step": 20,
      "train_speed(iter/s)": 0.010713
    },
    {
      "epoch": 0.04686192468619247,
      "grad_norm": 1.775608777999878,
      "learning_rate": 9.333333333333334e-05,
      "logits/chosen": -0.4235825836658478,
      "logits/rejected": -0.4997040629386902,
      "logps/chosen": -7.756269454956055,
      "logps/rejected": -22.05597496032715,
      "loss": 1.1610935926437378,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5124741792678833,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.056686438620090485,
      "rewards/margins": 0.15342777967453003,
      "rewards/rejected": -0.09674134850502014,
      "step": 21,
      "train_speed(iter/s)": 0.010711
    },
    {
      "epoch": 0.04909344490934449,
      "grad_norm": 1.6878494024276733,
      "learning_rate": 9.777777777777778e-05,
      "logits/chosen": -0.448173850774765,
      "logits/rejected": -0.46942824125289917,
      "logps/chosen": -9.774542808532715,
      "logps/rejected": -14.852673530578613,
      "loss": 1.1857895851135254,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5429187417030334,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.08716461062431335,
      "rewards/margins": 0.17264463007450104,
      "rewards/rejected": -0.08548003435134888,
      "step": 22,
      "train_speed(iter/s)": 0.01071
    },
    {
      "epoch": 0.051324965132496514,
      "grad_norm": 1.9283621311187744,
      "learning_rate": 0.00010222222222222222,
      "logits/chosen": -0.4297652244567871,
      "logits/rejected": -0.5464367866516113,
      "logps/chosen": -7.680075168609619,
      "logps/rejected": -20.64339828491211,
      "loss": 1.1528663635253906,
      "memory(GiB)": 46.81,
      "nll_loss": 0.48048245906829834,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.05369298905134201,
      "rewards/margins": 0.09629381448030472,
      "rewards/rejected": -0.04260082542896271,
      "step": 23,
      "train_speed(iter/s)": 0.01071
    },
    {
      "epoch": 0.05355648535564853,
      "grad_norm": 1.8897244930267334,
      "learning_rate": 0.00010666666666666667,
      "logits/chosen": -0.4073754847049713,
      "logits/rejected": -0.4649578034877777,
      "logps/chosen": -9.451826095581055,
      "logps/rejected": -20.540328979492188,
      "loss": 1.0510568618774414,
      "memory(GiB)": 46.81,
      "nll_loss": 0.431056946516037,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.055333852767944336,
      "rewards/margins": 0.23058435320854187,
      "rewards/rejected": -0.17525050044059753,
      "step": 24,
      "train_speed(iter/s)": 0.010709
    },
    {
      "epoch": 0.05578800557880056,
      "grad_norm": 1.968520998954773,
      "learning_rate": 0.00011111111111111112,
      "logits/chosen": -0.4085446000099182,
      "logits/rejected": -0.49927735328674316,
      "logps/chosen": -9.85863971710205,
      "logps/rejected": -21.732637405395508,
      "loss": 1.166273593902588,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5423365831375122,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03325103968381882,
      "rewards/margins": 0.2285088747739792,
      "rewards/rejected": -0.19525781273841858,
      "step": 25,
      "train_speed(iter/s)": 0.010709
    },
    {
      "epoch": 0.05801952580195258,
      "grad_norm": 3.0135302543640137,
      "learning_rate": 0.00011555555555555555,
      "logits/chosen": -0.44835054874420166,
      "logits/rejected": -0.5419439077377319,
      "logps/chosen": -4.6747965812683105,
      "logps/rejected": -18.435997009277344,
      "loss": 0.9960591793060303,
      "memory(GiB)": 46.81,
      "nll_loss": 0.39432889223098755,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.017948223277926445,
      "rewards/margins": 0.27569395303726196,
      "rewards/rejected": -0.29364219307899475,
      "step": 26,
      "train_speed(iter/s)": 0.010712
    },
    {
      "epoch": 0.0602510460251046,
      "grad_norm": 2.0994958877563477,
      "learning_rate": 0.00012,
      "logits/chosen": -0.4755466878414154,
      "logits/rejected": -0.513874888420105,
      "logps/chosen": -14.17818832397461,
      "logps/rejected": -15.192220687866211,
      "loss": 1.3128914833068848,
      "memory(GiB)": 46.81,
      "nll_loss": 0.732846736907959,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.11946821957826614,
      "rewards/margins": 0.27503034472465515,
      "rewards/rejected": -0.1555621325969696,
      "step": 27,
      "train_speed(iter/s)": 0.010714
    },
    {
      "epoch": 0.06248256624825663,
      "grad_norm": 2.619943857192993,
      "learning_rate": 0.00012444444444444444,
      "logits/chosen": -0.33525484800338745,
      "logits/rejected": -0.4649803340435028,
      "logps/chosen": -6.553568363189697,
      "logps/rejected": -23.942062377929688,
      "loss": 1.007144808769226,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4446461498737335,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07584571838378906,
      "rewards/margins": 0.3631606996059418,
      "rewards/rejected": -0.2873149812221527,
      "step": 28,
      "train_speed(iter/s)": 0.010711
    },
    {
      "epoch": 0.06471408647140865,
      "grad_norm": 2.1066770553588867,
      "learning_rate": 0.00012888888888888892,
      "logits/chosen": -0.37913525104522705,
      "logits/rejected": -0.4620123505592346,
      "logps/chosen": -7.321231842041016,
      "logps/rejected": -21.526382446289062,
      "loss": 1.0192487239837646,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4804428219795227,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.026389580219984055,
      "rewards/margins": 0.46746930480003357,
      "rewards/rejected": -0.441079705953598,
      "step": 29,
      "train_speed(iter/s)": 0.010712
    },
    {
      "epoch": 0.06694560669456066,
      "grad_norm": 1.8226919174194336,
      "learning_rate": 0.00013333333333333334,
      "logits/chosen": -0.3827905058860779,
      "logits/rejected": -0.48159855604171753,
      "logps/chosen": -6.776709079742432,
      "logps/rejected": -22.80392074584961,
      "loss": 1.0347448587417603,
      "memory(GiB)": 46.81,
      "nll_loss": 0.500119149684906,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.10474913567304611,
      "rewards/margins": 0.39701035618782043,
      "rewards/rejected": -0.2922612428665161,
      "step": 30,
      "train_speed(iter/s)": 0.010712
    },
    {
      "epoch": 0.06917712691771269,
      "grad_norm": 1.8681020736694336,
      "learning_rate": 0.0001377777777777778,
      "logits/chosen": -0.3609718978404999,
      "logits/rejected": -0.43842607736587524,
      "logps/chosen": -5.881144046783447,
      "logps/rejected": -21.236047744750977,
      "loss": 0.9905272126197815,
      "memory(GiB)": 46.81,
      "nll_loss": 0.43477141857147217,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.04010690003633499,
      "rewards/margins": 0.3823702037334442,
      "rewards/rejected": -0.34226328134536743,
      "step": 31,
      "train_speed(iter/s)": 0.010714
    },
    {
      "epoch": 0.07140864714086471,
      "grad_norm": 2.225977897644043,
      "learning_rate": 0.00014222222222222224,
      "logits/chosen": -0.39412301778793335,
      "logits/rejected": -0.4322904050350189,
      "logps/chosen": -10.140836715698242,
      "logps/rejected": -18.79208755493164,
      "loss": 1.0995820760726929,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5035545229911804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.04120786115527153,
      "rewards/margins": 0.2624339461326599,
      "rewards/rejected": -0.22122608125209808,
      "step": 32,
      "train_speed(iter/s)": 0.010716
    },
    {
      "epoch": 0.07364016736401674,
      "grad_norm": 2.788853406906128,
      "learning_rate": 0.00014666666666666666,
      "logits/chosen": -0.27406275272369385,
      "logits/rejected": -0.371414452791214,
      "logps/chosen": -12.10950756072998,
      "logps/rejected": -24.930038452148438,
      "loss": 1.2428903579711914,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6102337837219238,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.08664405345916748,
      "rewards/margins": 0.2861720621585846,
      "rewards/rejected": -0.3728161156177521,
      "step": 33,
      "train_speed(iter/s)": 0.010715
    },
    {
      "epoch": 0.07587168758716877,
      "grad_norm": 2.416369676589966,
      "learning_rate": 0.0001511111111111111,
      "logits/chosen": -0.24224025011062622,
      "logits/rejected": -0.3833686113357544,
      "logps/chosen": -6.028831958770752,
      "logps/rejected": -22.455066680908203,
      "loss": 0.9914643168449402,
      "memory(GiB)": 46.81,
      "nll_loss": 0.44179052114486694,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.011510152369737625,
      "rewards/margins": 0.5171858668327332,
      "rewards/rejected": -0.5056756734848022,
      "step": 34,
      "train_speed(iter/s)": 0.010714
    },
    {
      "epoch": 0.07810320781032078,
      "grad_norm": 4.029798984527588,
      "learning_rate": 0.00015555555555555556,
      "logits/chosen": -0.2615310847759247,
      "logits/rejected": -0.39800190925598145,
      "logps/chosen": -5.90588903427124,
      "logps/rejected": -24.642311096191406,
      "loss": 0.8372505307197571,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4036243259906769,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.03906700387597084,
      "rewards/margins": 0.8896325826644897,
      "rewards/rejected": -0.9286996126174927,
      "step": 35,
      "train_speed(iter/s)": 0.010715
    },
    {
      "epoch": 0.0803347280334728,
      "grad_norm": 2.4356110095977783,
      "learning_rate": 0.00016,
      "logits/chosen": -0.25296953320503235,
      "logits/rejected": -0.3325399160385132,
      "logps/chosen": -9.764960289001465,
      "logps/rejected": -23.14637565612793,
      "loss": 1.0952283143997192,
      "memory(GiB)": 46.81,
      "nll_loss": 0.641168475151062,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.002499673515558243,
      "rewards/margins": 0.8523399829864502,
      "rewards/rejected": -0.8498403429985046,
      "step": 36,
      "train_speed(iter/s)": 0.010716
    },
    {
      "epoch": 0.08256624825662483,
      "grad_norm": 1.9291634559631348,
      "learning_rate": 0.00016444444444444444,
      "logits/chosen": -0.20930960774421692,
      "logits/rejected": -0.3604508936405182,
      "logps/chosen": -2.847867488861084,
      "logps/rejected": -30.63514518737793,
      "loss": 0.6885566711425781,
      "memory(GiB)": 46.81,
      "nll_loss": 0.2891227900981903,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.061974309384822845,
      "rewards/margins": 1.0785088539123535,
      "rewards/rejected": -1.0165345668792725,
      "step": 37,
      "train_speed(iter/s)": 0.010715
    },
    {
      "epoch": 0.08479776847977685,
      "grad_norm": 1.6937510967254639,
      "learning_rate": 0.00016888888888888889,
      "logits/chosen": -0.2987847626209259,
      "logits/rejected": -0.37470635771751404,
      "logps/chosen": -6.199091911315918,
      "logps/rejected": -19.3011474609375,
      "loss": 0.8923996090888977,
      "memory(GiB)": 46.81,
      "nll_loss": 0.38914668560028076,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0021905135363340378,
      "rewards/margins": 0.6954408288002014,
      "rewards/rejected": -0.6932503581047058,
      "step": 38,
      "train_speed(iter/s)": 0.010716
    },
    {
      "epoch": 0.08702928870292886,
      "grad_norm": 2.106278896331787,
      "learning_rate": 0.00017333333333333334,
      "logits/chosen": -0.28487277030944824,
      "logits/rejected": -0.34711676836013794,
      "logps/chosen": -8.657167434692383,
      "logps/rejected": -26.564130783081055,
      "loss": 1.023146390914917,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5919384956359863,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05617165565490723,
      "rewards/margins": 1.1481283903121948,
      "rewards/rejected": -1.091956615447998,
      "step": 39,
      "train_speed(iter/s)": 0.010717
    },
    {
      "epoch": 0.08926080892608089,
      "grad_norm": 2.619154214859009,
      "learning_rate": 0.00017777777777777779,
      "logits/chosen": -0.2898882031440735,
      "logits/rejected": -0.3393169343471527,
      "logps/chosen": -8.001999855041504,
      "logps/rejected": -20.189369201660156,
      "loss": 0.9479650855064392,
      "memory(GiB)": 46.81,
      "nll_loss": 0.42854148149490356,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0008942075073719025,
      "rewards/margins": 0.7672775983810425,
      "rewards/rejected": -0.7663832902908325,
      "step": 40,
      "train_speed(iter/s)": 0.010719
    },
    {
      "epoch": 0.09149232914923291,
      "grad_norm": 2.770841360092163,
      "learning_rate": 0.00018222222222222224,
      "logits/chosen": -0.22048279643058777,
      "logits/rejected": -0.30986371636390686,
      "logps/chosen": -5.240083694458008,
      "logps/rejected": -22.307451248168945,
      "loss": 0.8702710866928101,
      "memory(GiB)": 46.81,
      "nll_loss": 0.39226293563842773,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.017199480906128883,
      "rewards/margins": 0.9372033476829529,
      "rewards/rejected": -0.9200038313865662,
      "step": 41,
      "train_speed(iter/s)": 0.010719
    },
    {
      "epoch": 0.09372384937238494,
      "grad_norm": 4.61620569229126,
      "learning_rate": 0.0001866666666666667,
      "logits/chosen": -0.30797824263572693,
      "logits/rejected": -0.3721832036972046,
      "logps/chosen": -10.481635093688965,
      "logps/rejected": -23.132564544677734,
      "loss": 1.2510943412780762,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6913026571273804,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.25621044635772705,
      "rewards/margins": 0.5585607886314392,
      "rewards/rejected": -0.814771294593811,
      "step": 42,
      "train_speed(iter/s)": 0.010721
    },
    {
      "epoch": 0.09595536959553697,
      "grad_norm": 3.806872844696045,
      "learning_rate": 0.00019111111111111114,
      "logits/chosen": -0.22567886114120483,
      "logits/rejected": -0.3761019706726074,
      "logps/chosen": -8.201981544494629,
      "logps/rejected": -34.75733184814453,
      "loss": 1.0068457126617432,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6396302580833435,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.12671321630477905,
      "rewards/margins": 1.7875899076461792,
      "rewards/rejected": -1.914302945137024,
      "step": 43,
      "train_speed(iter/s)": 0.010722
    },
    {
      "epoch": 0.09818688981868898,
      "grad_norm": 3.342564821243286,
      "learning_rate": 0.00019555555555555556,
      "logits/chosen": -0.259334534406662,
      "logits/rejected": -0.33293721079826355,
      "logps/chosen": -9.735747337341309,
      "logps/rejected": -34.48583221435547,
      "loss": 0.9845027327537537,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5914844870567322,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.2998116612434387,
      "rewards/margins": 1.7843880653381348,
      "rewards/rejected": -2.084199905395508,
      "step": 44,
      "train_speed(iter/s)": 0.010723
    },
    {
      "epoch": 0.100418410041841,
      "grad_norm": 3.4293951988220215,
      "learning_rate": 0.0002,
      "logits/chosen": -0.34163057804107666,
      "logits/rejected": -0.36223551630973816,
      "logps/chosen": -9.353399276733398,
      "logps/rejected": -21.651199340820312,
      "loss": 1.0366103649139404,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5141280889511108,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.003630978986620903,
      "rewards/margins": 0.7049190402030945,
      "rewards/rejected": -0.7085499167442322,
      "step": 45,
      "train_speed(iter/s)": 0.010724
    },
    {
      "epoch": 0.10264993026499303,
      "grad_norm": 2.1774137020111084,
      "learning_rate": 0.0001999993185874369,
      "logits/chosen": -0.3677416443824768,
      "logits/rejected": -0.3854161500930786,
      "logps/chosen": -11.536070823669434,
      "logps/rejected": -27.305158615112305,
      "loss": 0.9515228271484375,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5171921849250793,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09924840927124023,
      "rewards/margins": 1.116947889328003,
      "rewards/rejected": -1.2161962985992432,
      "step": 46,
      "train_speed(iter/s)": 0.010724
    },
    {
      "epoch": 0.10488145048814505,
      "grad_norm": 2.2629730701446533,
      "learning_rate": 0.00019999727435903407,
      "logits/chosen": -0.3420083522796631,
      "logits/rejected": -0.4473438858985901,
      "logps/chosen": -4.170689582824707,
      "logps/rejected": -21.129119873046875,
      "loss": 0.9114863276481628,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3966943621635437,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1746423840522766,
      "rewards/margins": 0.61202472448349,
      "rewards/rejected": -0.43738237023353577,
      "step": 47,
      "train_speed(iter/s)": 0.010725
    },
    {
      "epoch": 0.10711297071129706,
      "grad_norm": 1.6574047803878784,
      "learning_rate": 0.0001999938673426507,
      "logits/chosen": -0.31779757142066956,
      "logits/rejected": -0.4338604807853699,
      "logps/chosen": -5.825237274169922,
      "logps/rejected": -20.981552124023438,
      "loss": 0.9128789305686951,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4218422770500183,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1773630529642105,
      "rewards/margins": 0.718612551689148,
      "rewards/rejected": -0.5412494540214539,
      "step": 48,
      "train_speed(iter/s)": 0.010725
    },
    {
      "epoch": 0.10934449093444909,
      "grad_norm": 2.0683908462524414,
      "learning_rate": 0.00019998909758471852,
      "logits/chosen": -0.3863949179649353,
      "logits/rejected": -0.4458945393562317,
      "logps/chosen": -7.798675537109375,
      "logps/rejected": -21.307409286499023,
      "loss": 1.0268117189407349,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5051021575927734,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08097527176141739,
      "rewards/margins": 0.7529874444007874,
      "rewards/rejected": -0.672012209892273,
      "step": 49,
      "train_speed(iter/s)": 0.010726
    },
    {
      "epoch": 0.11157601115760112,
      "grad_norm": 2.338461399078369,
      "learning_rate": 0.000199982965150241,
      "logits/chosen": -0.34079253673553467,
      "logits/rejected": -0.3965541124343872,
      "logps/chosen": -11.766353607177734,
      "logps/rejected": -19.651931762695312,
      "loss": 1.1824036836624146,
      "memory(GiB)": 46.81,
      "nll_loss": 0.609149158000946,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.08129046857357025,
      "rewards/margins": 0.3777032196521759,
      "rewards/rejected": -0.4589936435222626,
      "step": 50,
      "train_speed(iter/s)": 0.010726
    },
    {
      "epoch": 0.11380753138075314,
      "grad_norm": 1.6744564771652222,
      "learning_rate": 0.00019997547012279244,
      "logits/chosen": -0.31765079498291016,
      "logits/rejected": -0.42205262184143066,
      "logps/chosen": -5.897686958312988,
      "logps/rejected": -25.42192268371582,
      "loss": 0.8473464846611023,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4084213376045227,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2017829716205597,
      "rewards/margins": 0.7832211256027222,
      "rewards/rejected": -0.5814382433891296,
      "step": 51,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.11603905160390517,
      "grad_norm": 1.8061268329620361,
      "learning_rate": 0.00019996661260451704,
      "logits/chosen": -0.32708457112312317,
      "logits/rejected": -0.41108813881874084,
      "logps/chosen": -8.663649559020996,
      "logps/rejected": -32.308448791503906,
      "loss": 0.8938776850700378,
      "memory(GiB)": 46.81,
      "nll_loss": 0.48662397265434265,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.01828554831445217,
      "rewards/margins": 1.3651036024093628,
      "rewards/rejected": -1.3833892345428467,
      "step": 52,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.11827057182705718,
      "grad_norm": 1.9129594564437866,
      "learning_rate": 0.00019995639271612722,
      "logits/chosen": -0.3344545364379883,
      "logits/rejected": -0.43980082869529724,
      "logps/chosen": -9.373285293579102,
      "logps/rejected": -30.374614715576172,
      "loss": 0.8187388181686401,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3988412916660309,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.04744601994752884,
      "rewards/margins": 1.1539989709854126,
      "rewards/rejected": -1.2014449834823608,
      "step": 53,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.1205020920502092,
      "grad_norm": 6.22075891494751,
      "learning_rate": 0.00019994481059690223,
      "logits/chosen": -0.3710540533065796,
      "logits/rejected": -0.45295441150665283,
      "logps/chosen": -11.353669166564941,
      "logps/rejected": -35.60136795043945,
      "loss": 1.3072545528411865,
      "memory(GiB)": 46.81,
      "nll_loss": 0.8566557168960571,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.1803397834300995,
      "rewards/margins": 2.1112091541290283,
      "rewards/rejected": -2.29154896736145,
      "step": 54,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.12273361227336123,
      "grad_norm": 3.538923501968384,
      "learning_rate": 0.00019993186640468604,
      "logits/chosen": -0.2791898250579834,
      "logits/rejected": -0.3996254801750183,
      "logps/chosen": -12.779634475708008,
      "logps/rejected": -37.21310043334961,
      "loss": 1.0243951082229614,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6127378940582275,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.261193186044693,
      "rewards/margins": 1.434675693511963,
      "rewards/rejected": -1.695868968963623,
      "step": 55,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.12496513249651325,
      "grad_norm": 2.5944247245788574,
      "learning_rate": 0.0001999175603158854,
      "logits/chosen": -0.2531765401363373,
      "logits/rejected": -0.4114147424697876,
      "logps/chosen": -9.740252494812012,
      "logps/rejected": -36.96112823486328,
      "loss": 0.8653287291526794,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5133413076400757,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.015189380384981632,
      "rewards/margins": 1.5309333801269531,
      "rewards/rejected": -1.5461229085922241,
      "step": 56,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.12719665271966527,
      "grad_norm": 3.273171901702881,
      "learning_rate": 0.0001999018925254673,
      "logits/chosen": -0.3509857654571533,
      "logits/rejected": -0.4661892056465149,
      "logps/chosen": -8.814093589782715,
      "logps/rejected": -30.390697479248047,
      "loss": 0.9130730032920837,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5733908414840698,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0046962546184659,
      "rewards/margins": 1.3072673082351685,
      "rewards/rejected": -1.3025707006454468,
      "step": 57,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.1294281729428173,
      "grad_norm": 2.717229127883911,
      "learning_rate": 0.00019988486324695628,
      "logits/chosen": -0.33554819226264954,
      "logits/rejected": -0.4281303584575653,
      "logps/chosen": -14.103944778442383,
      "logps/rejected": -32.3876838684082,
      "loss": 0.9715602397918701,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5424861311912537,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0524468794465065,
      "rewards/margins": 1.0883628129959106,
      "rewards/rejected": -1.1408096551895142,
      "step": 58,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.13165969316596932,
      "grad_norm": 2.887491226196289,
      "learning_rate": 0.00019986647271243163,
      "logits/chosen": -0.3923209011554718,
      "logits/rejected": -0.48743686079978943,
      "logps/chosen": -6.2308878898620605,
      "logps/rejected": -21.537296295166016,
      "loss": 0.9816198945045471,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4462883472442627,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06832532584667206,
      "rewards/margins": 0.5661296844482422,
      "rewards/rejected": -0.6344550251960754,
      "step": 59,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.13389121338912133,
      "grad_norm": 2.2311718463897705,
      "learning_rate": 0.00019984672117252423,
      "logits/chosen": -0.33507245779037476,
      "logits/rejected": -0.49380406737327576,
      "logps/chosen": -9.220368385314941,
      "logps/rejected": -37.96326446533203,
      "loss": 0.9401113986968994,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5612199902534485,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.06330323219299316,
      "rewards/margins": 1.686211347579956,
      "rewards/rejected": -1.622908115386963,
      "step": 60,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.13612273361227337,
      "grad_norm": 1.992959976196289,
      "learning_rate": 0.00019982560889641296,
      "logits/chosen": -0.35564690828323364,
      "logits/rejected": -0.5200420618057251,
      "logps/chosen": -3.4553239345550537,
      "logps/rejected": -32.64472198486328,
      "loss": 0.640341579914093,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3033585250377655,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08748181164264679,
      "rewards/margins": 1.5070304870605469,
      "rewards/rejected": -1.419548511505127,
      "step": 61,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.13835425383542538,
      "grad_norm": 3.40260648727417,
      "learning_rate": 0.00019980313617182127,
      "logits/chosen": -0.3457183837890625,
      "logits/rejected": -0.5498067736625671,
      "logps/chosen": -6.69366455078125,
      "logps/rejected": -37.40818405151367,
      "loss": 0.9436236619949341,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6088553667068481,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.10982945561408997,
      "rewards/margins": 1.797737956047058,
      "rewards/rejected": -1.9075673818588257,
      "step": 62,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.14058577405857742,
      "grad_norm": 4.374825954437256,
      "learning_rate": 0.00019977930330501308,
      "logits/chosen": -0.42058196663856506,
      "logits/rejected": -0.5126227736473083,
      "logps/chosen": -10.140219688415527,
      "logps/rejected": -33.032615661621094,
      "loss": 1.193029522895813,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6540992856025696,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.22300566732883453,
      "rewards/margins": 1.272392988204956,
      "rewards/rejected": -1.495398759841919,
      "step": 63,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.14281729428172943,
      "grad_norm": 4.756910800933838,
      "learning_rate": 0.0001997541106207887,
      "logits/chosen": -0.4631284475326538,
      "logits/rejected": -0.5332682728767395,
      "logps/chosen": -8.054980278015137,
      "logps/rejected": -31.029682159423828,
      "loss": 1.0884121656417847,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6862107515335083,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.1161859929561615,
      "rewards/margins": 1.4922480583190918,
      "rewards/rejected": -1.6084339618682861,
      "step": 64,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.14504881450488144,
      "grad_norm": 3.428164005279541,
      "learning_rate": 0.0001997275584624803,
      "logits/chosen": -0.3415633738040924,
      "logits/rejected": -0.49688977003097534,
      "logps/chosen": -7.313436031341553,
      "logps/rejected": -35.58869934082031,
      "loss": 0.957207977771759,
      "memory(GiB)": 46.81,
      "nll_loss": 0.590711772441864,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05026661604642868,
      "rewards/margins": 1.5162088871002197,
      "rewards/rejected": -1.4659425020217896,
      "step": 65,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.14728033472803348,
      "grad_norm": 3.1654562950134277,
      "learning_rate": 0.00019969964719194745,
      "logits/chosen": -0.4512020945549011,
      "logits/rejected": -0.5287590026855469,
      "logps/chosen": -8.630544662475586,
      "logps/rejected": -15.341949462890625,
      "loss": 1.1465851068496704,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5758541822433472,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06041776388883591,
      "rewards/margins": 0.51249760389328,
      "rewards/rejected": -0.45207977294921875,
      "step": 66,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.1495118549511855,
      "grad_norm": 2.0986812114715576,
      "learning_rate": 0.0001996703771895719,
      "logits/chosen": -0.5272831320762634,
      "logits/rejected": -0.517794668674469,
      "logps/chosen": -12.980257034301758,
      "logps/rejected": -18.752052307128906,
      "loss": 1.3610563278198242,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6797517538070679,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.026422299444675446,
      "rewards/margins": 0.5318213105201721,
      "rewards/rejected": -0.5053990483283997,
      "step": 67,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.15174337517433753,
      "grad_norm": 1.9081969261169434,
      "learning_rate": 0.00019963974885425266,
      "logits/chosen": -0.4080069661140442,
      "logits/rejected": -0.47813406586647034,
      "logps/chosen": -7.824826717376709,
      "logps/rejected": -18.751680374145508,
      "loss": 1.0740342140197754,
      "memory(GiB)": 46.81,
      "nll_loss": 0.45793265104293823,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.04187186062335968,
      "rewards/margins": 0.3901542127132416,
      "rewards/rejected": -0.3482823669910431,
      "step": 68,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.15397489539748954,
      "grad_norm": 1.700203537940979,
      "learning_rate": 0.0001996077626034003,
      "logits/chosen": -0.45641785860061646,
      "logits/rejected": -0.5060834884643555,
      "logps/chosen": -11.462532043457031,
      "logps/rejected": -16.27201271057129,
      "loss": 1.1681339740753174,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6015636920928955,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.136613130569458,
      "rewards/margins": 0.3738591969013214,
      "rewards/rejected": -0.2372460514307022,
      "step": 69,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.15620641562064155,
      "grad_norm": 1.9888875484466553,
      "learning_rate": 0.00019957441887293156,
      "logits/chosen": -0.4031111001968384,
      "logits/rejected": -0.47320762276649475,
      "logps/chosen": -7.325850009918213,
      "logps/rejected": -25.553611755371094,
      "loss": 1.002021074295044,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4237433671951294,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.013660086318850517,
      "rewards/margins": 0.49219831824302673,
      "rewards/rejected": -0.4785382151603699,
      "step": 70,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.1584379358437936,
      "grad_norm": 1.5379258394241333,
      "learning_rate": 0.0001995397181172631,
      "logits/chosen": -0.40455764532089233,
      "logits/rejected": -0.518138587474823,
      "logps/chosen": -3.8365478515625,
      "logps/rejected": -26.37249755859375,
      "loss": 0.8130152225494385,
      "memory(GiB)": 46.81,
      "nll_loss": 0.34526968002319336,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10106083005666733,
      "rewards/margins": 0.8268870115280151,
      "rewards/rejected": -0.7258262038230896,
      "step": 71,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.1606694560669456,
      "grad_norm": 1.612131953239441,
      "learning_rate": 0.0001995036608093056,
      "logits/chosen": -0.4558025002479553,
      "logits/rejected": -0.48799094557762146,
      "logps/chosen": -8.70871639251709,
      "logps/rejected": -17.577770233154297,
      "loss": 0.8906623125076294,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3765795826911926,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05890116095542908,
      "rewards/margins": 0.6036397814750671,
      "rewards/rejected": -0.5447385907173157,
      "step": 72,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.16290097629009762,
      "grad_norm": 1.3791937828063965,
      "learning_rate": 0.00019946624744045704,
      "logits/chosen": -0.4362650215625763,
      "logits/rejected": -0.48206600546836853,
      "logps/chosen": -10.413619995117188,
      "logps/rejected": -27.357479095458984,
      "loss": 0.8979863524436951,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3758661448955536,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.009980186820030212,
      "rewards/margins": 0.9504665732383728,
      "rewards/rejected": -0.960446834564209,
      "step": 73,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.16513249651324965,
      "grad_norm": 1.7471097707748413,
      "learning_rate": 0.00019942747852059632,
      "logits/chosen": -0.423688143491745,
      "logits/rejected": -0.49418017268180847,
      "logps/chosen": -8.749202728271484,
      "logps/rejected": -33.218692779541016,
      "loss": 0.8359287977218628,
      "memory(GiB)": 46.81,
      "nll_loss": 0.36860865354537964,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05798308178782463,
      "rewards/margins": 1.4300649166107178,
      "rewards/rejected": -1.4880479574203491,
      "step": 74,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 2.2621185779571533,
      "learning_rate": 0.00019938735457807592,
      "logits/chosen": -0.40228813886642456,
      "logits/rejected": -0.47656214237213135,
      "logps/chosen": -10.918057441711426,
      "logps/rejected": -30.61962890625,
      "loss": 0.9725239872932434,
      "memory(GiB)": 46.81,
      "nll_loss": 0.499444842338562,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06699477136135101,
      "rewards/margins": 0.938097357749939,
      "rewards/rejected": -1.0050921440124512,
      "step": 75,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.1695955369595537,
      "grad_norm": 2.474619150161743,
      "learning_rate": 0.00019934587615971506,
      "logits/chosen": -0.38186344504356384,
      "logits/rejected": -0.4598054587841034,
      "logps/chosen": -6.823958873748779,
      "logps/rejected": -29.09862518310547,
      "loss": 0.9761800765991211,
      "memory(GiB)": 46.81,
      "nll_loss": 0.48571133613586426,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.05072644352912903,
      "rewards/margins": 0.8625801205635071,
      "rewards/rejected": -0.9133066534996033,
      "step": 76,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.17182705718270572,
      "grad_norm": 1.823959231376648,
      "learning_rate": 0.00019930304383079204,
      "logits/chosen": -0.4149280786514282,
      "logits/rejected": -0.5062562227249146,
      "logps/chosen": -8.54432201385498,
      "logps/rejected": -29.242015838623047,
      "loss": 0.8722183108329773,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4606979191303253,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01995794102549553,
      "rewards/margins": 1.4311785697937012,
      "rewards/rejected": -1.4112205505371094,
      "step": 77,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.17405857740585773,
      "grad_norm": 1.793901801109314,
      "learning_rate": 0.00019925885817503663,
      "logits/chosen": -0.4355413317680359,
      "logits/rejected": -0.5245873332023621,
      "logps/chosen": -4.893879413604736,
      "logps/rejected": -25.42880630493164,
      "loss": 0.7558175921440125,
      "memory(GiB)": 46.81,
      "nll_loss": 0.38899344205856323,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08300305157899857,
      "rewards/margins": 1.0970323085784912,
      "rewards/rejected": -1.0140292644500732,
      "step": 78,
      "train_speed(iter/s)": 0.010696
    },
    {
      "epoch": 0.17629009762900977,
      "grad_norm": 2.114241123199463,
      "learning_rate": 0.00019921331979462197,
      "logits/chosen": -0.3358357548713684,
      "logits/rejected": -0.47999072074890137,
      "logps/chosen": -5.950570583343506,
      "logps/rejected": -28.20298957824707,
      "loss": 0.8057999610900879,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4144690930843353,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05946407839655876,
      "rewards/margins": 1.0457231998443604,
      "rewards/rejected": -0.9862591028213501,
      "step": 79,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.17852161785216178,
      "grad_norm": 2.7315855026245117,
      "learning_rate": 0.00019916642931015662,
      "logits/chosen": -0.43205446004867554,
      "logits/rejected": -0.5006397366523743,
      "logps/chosen": -7.170259952545166,
      "logps/rejected": -22.804529190063477,
      "loss": 1.0863537788391113,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6158946752548218,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.03549172729253769,
      "rewards/margins": 0.8894610404968262,
      "rewards/rejected": -0.9249527454376221,
      "step": 80,
      "train_speed(iter/s)": 0.010698
    },
    {
      "epoch": 0.18075313807531382,
      "grad_norm": 2.479121685028076,
      "learning_rate": 0.00019911818736067586,
      "logits/chosen": -0.40614452958106995,
      "logits/rejected": -0.5118440389633179,
      "logps/chosen": -4.780482769012451,
      "logps/rejected": -22.085041046142578,
      "loss": 0.7858155965805054,
      "memory(GiB)": 46.81,
      "nll_loss": 0.44690829515457153,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21181246638298035,
      "rewards/margins": 1.2262333631515503,
      "rewards/rejected": -1.0144208669662476,
      "step": 81,
      "train_speed(iter/s)": 0.010699
    },
    {
      "epoch": 0.18298465829846583,
      "grad_norm": 1.6746541261672974,
      "learning_rate": 0.00019906859460363307,
      "logits/chosen": -0.4001500606536865,
      "logits/rejected": -0.45929989218711853,
      "logps/chosen": -7.395153045654297,
      "logps/rejected": -25.426557540893555,
      "loss": 0.7357066869735718,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4178068935871124,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16341763734817505,
      "rewards/margins": 1.525546669960022,
      "rewards/rejected": -1.3621290922164917,
      "step": 82,
      "train_speed(iter/s)": 0.0107
    },
    {
      "epoch": 0.18521617852161784,
      "grad_norm": 4.773955821990967,
      "learning_rate": 0.00019901765171489086,
      "logits/chosen": -0.30332040786743164,
      "logits/rejected": -0.4577845633029938,
      "logps/chosen": -10.913299560546875,
      "logps/rejected": -46.28175354003906,
      "loss": 1.1305065155029297,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7709158658981323,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.34224939346313477,
      "rewards/margins": 2.636122703552246,
      "rewards/rejected": -2.9783718585968018,
      "step": 83,
      "train_speed(iter/s)": 0.0107
    },
    {
      "epoch": 0.18744769874476988,
      "grad_norm": 2.5695321559906006,
      "learning_rate": 0.0001989653593887117,
      "logits/chosen": -0.3644600212574005,
      "logits/rejected": -0.43475809693336487,
      "logps/chosen": -6.348275661468506,
      "logps/rejected": -29.636547088623047,
      "loss": 0.8502273559570312,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4918310344219208,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07552118599414825,
      "rewards/margins": 1.7124391794204712,
      "rewards/rejected": -1.636918067932129,
      "step": 84,
      "train_speed(iter/s)": 0.010702
    },
    {
      "epoch": 0.1896792189679219,
      "grad_norm": 3.117232322692871,
      "learning_rate": 0.00019891171833774854,
      "logits/chosen": -0.31497788429260254,
      "logits/rejected": -0.40428590774536133,
      "logps/chosen": -10.195795059204102,
      "logps/rejected": -43.74475860595703,
      "loss": 1.0120048522949219,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6030641198158264,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.17749960720539093,
      "rewards/margins": 2.177133083343506,
      "rewards/rejected": -2.35463285446167,
      "step": 85,
      "train_speed(iter/s)": 0.010701
    },
    {
      "epoch": 0.19191073919107393,
      "grad_norm": 2.5167582035064697,
      "learning_rate": 0.00019885672929303508,
      "logits/chosen": -0.3179773986339569,
      "logits/rejected": -0.42796629667282104,
      "logps/chosen": -10.96944522857666,
      "logps/rejected": -40.80889129638672,
      "loss": 0.8473846912384033,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5096143484115601,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.08502935618162155,
      "rewards/margins": 2.197641134262085,
      "rewards/rejected": -2.282670259475708,
      "step": 86,
      "train_speed(iter/s)": 0.010702
    },
    {
      "epoch": 0.19414225941422594,
      "grad_norm": 2.2492237091064453,
      "learning_rate": 0.00019880039300397591,
      "logits/chosen": -0.4064781665802002,
      "logits/rejected": -0.4387715458869934,
      "logps/chosen": -7.350955009460449,
      "logps/rejected": -30.075700759887695,
      "loss": 0.9061151742935181,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5411654710769653,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04652661085128784,
      "rewards/margins": 1.5254417657852173,
      "rewards/rejected": -1.4789149761199951,
      "step": 87,
      "train_speed(iter/s)": 0.010703
    },
    {
      "epoch": 0.19637377963737795,
      "grad_norm": 1.698593258857727,
      "learning_rate": 0.00019874271023833604,
      "logits/chosen": -0.36257320642471313,
      "logits/rejected": -0.4434874653816223,
      "logps/chosen": -8.048574447631836,
      "logps/rejected": -30.737728118896484,
      "loss": 0.8314277529716492,
      "memory(GiB)": 46.81,
      "nll_loss": 0.47138625383377075,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10594448447227478,
      "rewards/margins": 1.524275541305542,
      "rewards/rejected": -1.4183311462402344,
      "step": 88,
      "train_speed(iter/s)": 0.010703
    },
    {
      "epoch": 0.19860529986053,
      "grad_norm": 3.0257344245910645,
      "learning_rate": 0.00019868368178223075,
      "logits/chosen": -0.3720957636833191,
      "logits/rejected": -0.45681309700012207,
      "logps/chosen": -9.780877113342285,
      "logps/rejected": -23.92654037475586,
      "loss": 1.1227757930755615,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6208180785179138,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.049955643713474274,
      "rewards/margins": 0.7902018427848816,
      "rewards/rejected": -0.8401575088500977,
      "step": 89,
      "train_speed(iter/s)": 0.010704
    },
    {
      "epoch": 0.200836820083682,
      "grad_norm": 1.996423363685608,
      "learning_rate": 0.00019862330844011466,
      "logits/chosen": -0.4094502031803131,
      "logits/rejected": -0.5237057209014893,
      "logps/chosen": -7.119333744049072,
      "logps/rejected": -27.10557746887207,
      "loss": 0.9298258423805237,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5608960390090942,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13814933598041534,
      "rewards/margins": 1.2807323932647705,
      "rewards/rejected": -1.1425830125808716,
      "step": 90,
      "train_speed(iter/s)": 0.010705
    },
    {
      "epoch": 0.20306834030683404,
      "grad_norm": 2.4107954502105713,
      "learning_rate": 0.00019856159103477086,
      "logits/chosen": -0.38746166229248047,
      "logits/rejected": -0.488497257232666,
      "logps/chosen": -4.5973896980285645,
      "logps/rejected": -27.419519424438477,
      "loss": 0.710839033126831,
      "memory(GiB)": 46.81,
      "nll_loss": 0.30818992853164673,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.038327932357788086,
      "rewards/margins": 1.1028857231140137,
      "rewards/rejected": -1.064557671546936,
      "step": 91,
      "train_speed(iter/s)": 0.010705
    },
    {
      "epoch": 0.20529986052998606,
      "grad_norm": 1.7134486436843872,
      "learning_rate": 0.00019849853040729962,
      "logits/chosen": -0.39711794257164,
      "logits/rejected": -0.5056676268577576,
      "logps/chosen": -3.698580265045166,
      "logps/rejected": -35.89771270751953,
      "loss": 0.6057847142219543,
      "memory(GiB)": 46.81,
      "nll_loss": 0.28385108709335327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.012411292642354965,
      "rewards/margins": 1.7848464250564575,
      "rewards/rejected": -1.7724350690841675,
      "step": 92,
      "train_speed(iter/s)": 0.010705
    },
    {
      "epoch": 0.20753138075313807,
      "grad_norm": 3.236150026321411,
      "learning_rate": 0.00019843412741710705,
      "logits/chosen": -0.4137122631072998,
      "logits/rejected": -0.5312527418136597,
      "logps/chosen": -8.622723579406738,
      "logps/rejected": -32.65446853637695,
      "loss": 0.923988938331604,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5595443248748779,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.008658979088068008,
      "rewards/margins": 1.6513465642929077,
      "rewards/rejected": -1.6600055694580078,
      "step": 93,
      "train_speed(iter/s)": 0.010706
    },
    {
      "epoch": 0.2097629009762901,
      "grad_norm": 3.0684919357299805,
      "learning_rate": 0.00019836838294189327,
      "logits/chosen": -0.30109331011772156,
      "logits/rejected": -0.5258408784866333,
      "logps/chosen": -4.7982635498046875,
      "logps/rejected": -46.58249282836914,
      "loss": 0.6704127788543701,
      "memory(GiB)": 46.81,
      "nll_loss": 0.41316378116607666,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.022386539727449417,
      "rewards/margins": 2.3209354877471924,
      "rewards/rejected": -2.3433220386505127,
      "step": 94,
      "train_speed(iter/s)": 0.010705
    },
    {
      "epoch": 0.21199442119944212,
      "grad_norm": 1.8835054636001587,
      "learning_rate": 0.00019830129787764048,
      "logits/chosen": -0.41429489850997925,
      "logits/rejected": -0.5583072304725647,
      "logps/chosen": -7.361013412475586,
      "logps/rejected": -39.35795211791992,
      "loss": 0.6725447773933411,
      "memory(GiB)": 46.81,
      "nll_loss": 0.368007630109787,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1507667899131775,
      "rewards/margins": 2.0595011711120605,
      "rewards/rejected": -2.210268020629883,
      "step": 95,
      "train_speed(iter/s)": 0.010706
    },
    {
      "epoch": 0.21422594142259413,
      "grad_norm": 2.3120245933532715,
      "learning_rate": 0.00019823287313860087,
      "logits/chosen": -0.372950941324234,
      "logits/rejected": -0.49759799242019653,
      "logps/chosen": -11.2495756149292,
      "logps/rejected": -34.734066009521484,
      "loss": 1.1070480346679688,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6910135746002197,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.07585638761520386,
      "rewards/margins": 1.6244981288909912,
      "rewards/rejected": -1.7003545761108398,
      "step": 96,
      "train_speed(iter/s)": 0.010706
    },
    {
      "epoch": 0.21645746164574617,
      "grad_norm": 2.277174711227417,
      "learning_rate": 0.00019816310965728388,
      "logits/chosen": -0.4299129247665405,
      "logits/rejected": -0.4980982542037964,
      "logps/chosen": -9.542954444885254,
      "logps/rejected": -25.756847381591797,
      "loss": 0.9643784165382385,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4663383364677429,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.04499448090791702,
      "rewards/margins": 1.144527554512024,
      "rewards/rejected": -1.1895220279693604,
      "step": 97,
      "train_speed(iter/s)": 0.010706
    },
    {
      "epoch": 0.21868898186889818,
      "grad_norm": 1.9245892763137817,
      "learning_rate": 0.00019809200838444383,
      "logits/chosen": -0.4750334322452545,
      "logits/rejected": -0.5499654412269592,
      "logps/chosen": -7.279554843902588,
      "logps/rejected": -27.49015998840332,
      "loss": 0.7538543343544006,
      "memory(GiB)": 46.81,
      "nll_loss": 0.37693145871162415,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19298376142978668,
      "rewards/margins": 1.2535287141799927,
      "rewards/rejected": -1.0605449676513672,
      "step": 98,
      "train_speed(iter/s)": 0.010707
    },
    {
      "epoch": 0.22092050209205022,
      "grad_norm": 2.6517419815063477,
      "learning_rate": 0.0001980195702890667,
      "logits/chosen": -0.3958469033241272,
      "logits/rejected": -0.5089414715766907,
      "logps/chosen": -7.152680397033691,
      "logps/rejected": -34.461917877197266,
      "loss": 0.8412438035011292,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3738154470920563,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.022486146539449692,
      "rewards/margins": 1.67362380027771,
      "rewards/rejected": -1.6961098909378052,
      "step": 99,
      "train_speed(iter/s)": 0.010708
    },
    {
      "epoch": 0.22315202231520223,
      "grad_norm": 2.8246142864227295,
      "learning_rate": 0.00019794579635835704,
      "logits/chosen": -0.4292357861995697,
      "logits/rejected": -0.5326910614967346,
      "logps/chosen": -7.5378594398498535,
      "logps/rejected": -21.57219696044922,
      "loss": 1.0386667251586914,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5401602387428284,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06580009311437607,
      "rewards/margins": 0.625900149345398,
      "rewards/rejected": -0.5601000189781189,
      "step": 100,
      "train_speed(iter/s)": 0.010708
    },
    {
      "epoch": 0.22538354253835424,
      "grad_norm": 2.5541062355041504,
      "learning_rate": 0.00019787068759772458,
      "logits/chosen": -0.2897275686264038,
      "logits/rejected": -0.469921737909317,
      "logps/chosen": -5.731183052062988,
      "logps/rejected": -42.3974494934082,
      "loss": 0.6600039005279541,
      "memory(GiB)": 46.81,
      "nll_loss": 0.34703782200813293,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03177756816148758,
      "rewards/margins": 2.005840539932251,
      "rewards/rejected": -1.9740626811981201,
      "step": 101,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.22761506276150628,
      "grad_norm": 1.7807402610778809,
      "learning_rate": 0.0001977942450307703,
      "logits/chosen": -0.4453420042991638,
      "logits/rejected": -0.5632447004318237,
      "logps/chosen": -2.4129638671875,
      "logps/rejected": -25.95719337463379,
      "loss": 0.5793961882591248,
      "memory(GiB)": 46.81,
      "nll_loss": 0.23024141788482666,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14239919185638428,
      "rewards/margins": 1.4281731843948364,
      "rewards/rejected": -1.2857739925384521,
      "step": 102,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.2298465829846583,
      "grad_norm": 2.2132644653320312,
      "learning_rate": 0.00019771646969927276,
      "logits/chosen": -0.35926875472068787,
      "logits/rejected": -0.520535945892334,
      "logps/chosen": -13.426995277404785,
      "logps/rejected": -45.62168884277344,
      "loss": 1.015347957611084,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7125861644744873,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.29009538888931274,
      "rewards/margins": 2.0907297134399414,
      "rewards/rejected": -2.3808252811431885,
      "step": 103,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.23207810320781033,
      "grad_norm": 3.2295114994049072,
      "learning_rate": 0.00019763736266317373,
      "logits/chosen": -0.40255796909332275,
      "logits/rejected": -0.48326143622398376,
      "logps/chosen": -11.606037139892578,
      "logps/rejected": -31.71380043029785,
      "loss": 1.1166499853134155,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6979129910469055,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08108239620923996,
      "rewards/margins": 1.4596813917160034,
      "rewards/rejected": -1.5407637357711792,
      "step": 104,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.23430962343096234,
      "grad_norm": 10.526008605957031,
      "learning_rate": 0.00019755692500056377,
      "logits/chosen": -0.39488697052001953,
      "logits/rejected": -0.4993595480918884,
      "logps/chosen": -8.515541076660156,
      "logps/rejected": -43.509639739990234,
      "loss": 0.9588133096694946,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6071394085884094,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.21072883903980255,
      "rewards/margins": 2.291653633117676,
      "rewards/rejected": -2.502382278442383,
      "step": 105,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.23654114365411436,
      "grad_norm": 4.369258880615234,
      "learning_rate": 0.0001974751578076675,
      "logits/chosen": -0.47596731781959534,
      "logits/rejected": -0.567024827003479,
      "logps/chosen": -9.769340515136719,
      "logps/rejected": -36.51396560668945,
      "loss": 1.1472177505493164,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7814697623252869,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07862536609172821,
      "rewards/margins": 1.9599454402923584,
      "rewards/rejected": -2.0385708808898926,
      "step": 106,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.2387726638772664,
      "grad_norm": 2.6377806663513184,
      "learning_rate": 0.0001973920621988288,
      "logits/chosen": -0.4363410472869873,
      "logits/rejected": -0.5180681943893433,
      "logps/chosen": -8.525186538696289,
      "logps/rejected": -36.119422912597656,
      "loss": 0.8840208053588867,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5660971999168396,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.11222146451473236,
      "rewards/margins": 2.031522035598755,
      "rewards/rejected": -2.1437435150146484,
      "step": 107,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.2410041841004184,
      "grad_norm": 3.5556135177612305,
      "learning_rate": 0.00019730763930649557,
      "logits/chosen": -0.4549260139465332,
      "logits/rejected": -0.5709996223449707,
      "logps/chosen": -6.807540416717529,
      "logps/rejected": -35.386817932128906,
      "loss": 1.0365853309631348,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6718467473983765,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.010064257308840752,
      "rewards/margins": 2.010096549987793,
      "rewards/rejected": -2.0201611518859863,
      "step": 108,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.24323570432357045,
      "grad_norm": 2.9586894512176514,
      "learning_rate": 0.0001972218902812041,
      "logits/chosen": -0.5169477462768555,
      "logits/rejected": -0.5766913890838623,
      "logps/chosen": -9.75178050994873,
      "logps/rejected": -26.94969940185547,
      "loss": 1.03122079372406,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6585339307785034,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.047493062913417816,
      "rewards/margins": 1.4683667421340942,
      "rewards/rejected": -1.5158597230911255,
      "step": 109,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.24546722454672246,
      "grad_norm": 2.40643310546875,
      "learning_rate": 0.0001971348162915637,
      "logits/chosen": -0.40012210607528687,
      "logits/rejected": -0.5228334665298462,
      "logps/chosen": -12.064148902893066,
      "logps/rejected": -42.65177917480469,
      "loss": 1.028669834136963,
      "memory(GiB)": 46.81,
      "nll_loss": 0.584478497505188,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0045367032289505005,
      "rewards/margins": 1.1938111782073975,
      "rewards/rejected": -1.189274549484253,
      "step": 110,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.24769874476987447,
      "grad_norm": 1.9528326988220215,
      "learning_rate": 0.0001970464185242406,
      "logits/chosen": -0.48454731702804565,
      "logits/rejected": -0.5713267922401428,
      "logps/chosen": -8.57382869720459,
      "logps/rejected": -28.3697452545166,
      "loss": 0.9442793130874634,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5884312987327576,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.007765632122755051,
      "rewards/margins": 1.1733622550964355,
      "rewards/rejected": -1.181127905845642,
      "step": 111,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.2499302649930265,
      "grad_norm": 1.6935343742370605,
      "learning_rate": 0.00019695669818394176,
      "logits/chosen": -0.49247032403945923,
      "logits/rejected": -0.5529605150222778,
      "logps/chosen": -6.149413108825684,
      "logps/rejected": -26.09927749633789,
      "loss": 0.8117169141769409,
      "memory(GiB)": 46.81,
      "nll_loss": 0.36330851912498474,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.11475833505392075,
      "rewards/margins": 0.9903795719146729,
      "rewards/rejected": -0.8756212592124939,
      "step": 112,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.25216178521617855,
      "grad_norm": 1.7400895357131958,
      "learning_rate": 0.00019686565649339852,
      "logits/chosen": -0.47781193256378174,
      "logits/rejected": -0.541789174079895,
      "logps/chosen": -7.246283054351807,
      "logps/rejected": -28.977123260498047,
      "loss": 0.8169933557510376,
      "memory(GiB)": 46.81,
      "nll_loss": 0.46227994561195374,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1930399090051651,
      "rewards/margins": 1.5713894367218018,
      "rewards/rejected": -1.3783494234085083,
      "step": 113,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.25439330543933053,
      "grad_norm": 2.0717272758483887,
      "learning_rate": 0.0001967732946933499,
      "logits/chosen": -0.48887836933135986,
      "logits/rejected": -0.5135952234268188,
      "logps/chosen": -9.175788879394531,
      "logps/rejected": -19.074743270874023,
      "loss": 1.0894757509231567,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5358699560165405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.011165066622197628,
      "rewards/margins": 0.7140389084815979,
      "rewards/rejected": -0.7252039313316345,
      "step": 114,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.25662482566248257,
      "grad_norm": 1.8609806299209595,
      "learning_rate": 0.00019667961404252573,
      "logits/chosen": -0.46024733781814575,
      "logits/rejected": -0.5330649018287659,
      "logps/chosen": -6.905147552490234,
      "logps/rejected": -30.54239273071289,
      "loss": 0.8680601119995117,
      "memory(GiB)": 46.81,
      "nll_loss": 0.44586634635925293,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03578485921025276,
      "rewards/margins": 1.5293731689453125,
      "rewards/rejected": -1.4935882091522217,
      "step": 115,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.2588563458856346,
      "grad_norm": 1.66657292842865,
      "learning_rate": 0.00019658461581762948,
      "logits/chosen": -0.430117666721344,
      "logits/rejected": -0.5585033297538757,
      "logps/chosen": -5.328842639923096,
      "logps/rejected": -37.261329650878906,
      "loss": 0.6072223782539368,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3235556483268738,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11161011457443237,
      "rewards/margins": 2.0768401622772217,
      "rewards/rejected": -1.965229868888855,
      "step": 116,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.2610878661087866,
      "grad_norm": 1.51878821849823,
      "learning_rate": 0.0001964883013133208,
      "logits/chosen": -0.419185996055603,
      "logits/rejected": -0.5297868251800537,
      "logps/chosen": -8.04638385772705,
      "logps/rejected": -37.03801727294922,
      "loss": 0.7135862708091736,
      "memory(GiB)": 46.81,
      "nll_loss": 0.40418556332588196,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.003706634044647217,
      "rewards/margins": 2.0922625064849854,
      "rewards/rejected": -2.0959692001342773,
      "step": 117,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.26331938633193863,
      "grad_norm": 1.7501200437545776,
      "learning_rate": 0.00019639067184219796,
      "logits/chosen": -0.4789353609085083,
      "logits/rejected": -0.5624681711196899,
      "logps/chosen": -2.698453426361084,
      "logps/rejected": -30.66053009033203,
      "loss": 0.4839392304420471,
      "memory(GiB)": 46.81,
      "nll_loss": 0.23937827348709106,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1991066038608551,
      "rewards/margins": 2.1478512287139893,
      "rewards/rejected": -1.948744773864746,
      "step": 118,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.26555090655509067,
      "grad_norm": 5.462699890136719,
      "learning_rate": 0.00019629172873477995,
      "logits/chosen": -0.4896751642227173,
      "logits/rejected": -0.6056660413742065,
      "logps/chosen": -9.186168670654297,
      "logps/rejected": -43.67456817626953,
      "loss": 1.0974973440170288,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7505476474761963,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.26940032839775085,
      "rewards/margins": 2.277892589569092,
      "rewards/rejected": -2.547292709350586,
      "step": 119,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.26778242677824265,
      "grad_norm": 2.521127462387085,
      "learning_rate": 0.00019619147333948823,
      "logits/chosen": -0.40272024273872375,
      "logits/rejected": -0.6465494632720947,
      "logps/chosen": -4.102967262268066,
      "logps/rejected": -46.14149475097656,
      "loss": 0.6112779974937439,
      "memory(GiB)": 46.81,
      "nll_loss": 0.35009804368019104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06559442728757858,
      "rewards/margins": 2.652864456176758,
      "rewards/rejected": -2.718458652496338,
      "step": 120,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.2700139470013947,
      "grad_norm": 6.966860771179199,
      "learning_rate": 0.00019608990702262862,
      "logits/chosen": -0.46823495626449585,
      "logits/rejected": -0.5669148564338684,
      "logps/chosen": -11.427492141723633,
      "logps/rejected": -41.708621978759766,
      "loss": 0.9116367101669312,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5967401266098022,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1262148916721344,
      "rewards/margins": 2.7404427528381348,
      "rewards/rejected": -2.8666577339172363,
      "step": 121,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.27224546722454673,
      "grad_norm": 3.0951266288757324,
      "learning_rate": 0.0001959870311683723,
      "logits/chosen": -0.4486353099346161,
      "logits/rejected": -0.5479130744934082,
      "logps/chosen": -12.51127815246582,
      "logps/rejected": -38.43931198120117,
      "loss": 0.9770362377166748,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5739753246307373,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2812083661556244,
      "rewards/margins": 1.7542171478271484,
      "rewards/rejected": -2.0354256629943848,
      "step": 122,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.2744769874476987,
      "grad_norm": 2.664623498916626,
      "learning_rate": 0.00019588284717873738,
      "logits/chosen": -0.48905450105667114,
      "logits/rejected": -0.5784589052200317,
      "logps/chosen": -11.445174217224121,
      "logps/rejected": -45.63172149658203,
      "loss": 0.8187350630760193,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5693225860595703,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07134091854095459,
      "rewards/margins": 2.74631929397583,
      "rewards/rejected": -2.8176605701446533,
      "step": 123,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.27670850767085076,
      "grad_norm": 1.5270544290542603,
      "learning_rate": 0.00019577735647356928,
      "logits/chosen": -0.42984533309936523,
      "logits/rejected": -0.5359001159667969,
      "logps/chosen": -6.653199195861816,
      "logps/rejected": -29.57577133178711,
      "loss": 0.7354912757873535,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3922813832759857,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1660708487033844,
      "rewards/margins": 1.5390381813049316,
      "rewards/rejected": -1.3729673624038696,
      "step": 124,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.2789400278940028,
      "grad_norm": 3.5429413318634033,
      "learning_rate": 0.00019567056049052203,
      "logits/chosen": -0.3107150197029114,
      "logits/rejected": -0.4991438388824463,
      "logps/chosen": -5.079807758331299,
      "logps/rejected": -40.358970642089844,
      "loss": 0.6619258522987366,
      "memory(GiB)": 46.81,
      "nll_loss": 0.36486440896987915,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.030078772455453873,
      "rewards/margins": 1.9852522611618042,
      "rewards/rejected": -1.955173373222351,
      "step": 125,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.28117154811715483,
      "grad_norm": 1.961870551109314,
      "learning_rate": 0.00019556246068503795,
      "logits/chosen": -0.44410189986228943,
      "logits/rejected": -0.4920841455459595,
      "logps/chosen": -11.91314697265625,
      "logps/rejected": -31.1485595703125,
      "loss": 0.8891083598136902,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5505969524383545,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.03779969736933708,
      "rewards/margins": 1.8280887603759766,
      "rewards/rejected": -1.790289044380188,
      "step": 126,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.2834030683403068,
      "grad_norm": 2.261699676513672,
      "learning_rate": 0.00019545305853032848,
      "logits/chosen": -0.3864215016365051,
      "logits/rejected": -0.4863501191139221,
      "logps/chosen": -7.286771774291992,
      "logps/rejected": -30.703035354614258,
      "loss": 0.8944107294082642,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5101214051246643,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.018673699349164963,
      "rewards/margins": 1.4196372032165527,
      "rewards/rejected": -1.400963544845581,
      "step": 127,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.28563458856345886,
      "grad_norm": 1.475645899772644,
      "learning_rate": 0.0001953423555173536,
      "logits/chosen": -0.36042094230651855,
      "logits/rejected": -0.5007886290550232,
      "logps/chosen": -5.484119892120361,
      "logps/rejected": -41.732566833496094,
      "loss": 0.587951123714447,
      "memory(GiB)": 46.81,
      "nll_loss": 0.33234262466430664,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.022678524255752563,
      "rewards/margins": 1.7825675010681152,
      "rewards/rejected": -1.7598888874053955,
      "step": 128,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.2878661087866109,
      "grad_norm": 1.8218269348144531,
      "learning_rate": 0.0001952303531548018,
      "logits/chosen": -0.33974117040634155,
      "logits/rejected": -0.44949695467948914,
      "logps/chosen": -5.385599136352539,
      "logps/rejected": -34.09246063232422,
      "loss": 0.566850483417511,
      "memory(GiB)": 46.81,
      "nll_loss": 0.25613701343536377,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12256469577550888,
      "rewards/margins": 1.9903541803359985,
      "rewards/rejected": -1.8677895069122314,
      "step": 129,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.2900976290097629,
      "grad_norm": 1.8668869733810425,
      "learning_rate": 0.00019511705296906945,
      "logits/chosen": -0.3487342596054077,
      "logits/rejected": -0.4806521534919739,
      "logps/chosen": -4.425655841827393,
      "logps/rejected": -30.515653610229492,
      "loss": 0.5262651443481445,
      "memory(GiB)": 46.81,
      "nll_loss": 0.25739917159080505,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12762486934661865,
      "rewards/margins": 1.6179413795471191,
      "rewards/rejected": -1.4903165102005005,
      "step": 130,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.2923291492329149,
      "grad_norm": 2.4468886852264404,
      "learning_rate": 0.00019500245650423987,
      "logits/chosen": -0.3514082133769989,
      "logits/rejected": -0.4678734242916107,
      "logps/chosen": -9.789356231689453,
      "logps/rejected": -43.26856231689453,
      "loss": 0.8514004349708557,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5882378816604614,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.00511053204536438,
      "rewards/margins": 2.595022678375244,
      "rewards/rejected": -2.600132942199707,
      "step": 131,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.29456066945606696,
      "grad_norm": 1.4899861812591553,
      "learning_rate": 0.00019488656532206256,
      "logits/chosen": -0.32015788555145264,
      "logits/rejected": -0.5057851672172546,
      "logps/chosen": -5.5769572257995605,
      "logps/rejected": -39.262237548828125,
      "loss": 0.5258051156997681,
      "memory(GiB)": 46.81,
      "nll_loss": 0.2577199935913086,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.012265877798199654,
      "rewards/margins": 1.970078945159912,
      "rewards/rejected": -1.957813024520874,
      "step": 132,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.29679218967921894,
      "grad_norm": 2.7044637203216553,
      "learning_rate": 0.00019476938100193166,
      "logits/chosen": -0.3656051754951477,
      "logits/rejected": -0.4236447215080261,
      "logps/chosen": -10.65760612487793,
      "logps/rejected": -38.63339614868164,
      "loss": 0.9246849417686462,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6106225848197937,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.030058247968554497,
      "rewards/margins": 2.29494047164917,
      "rewards/rejected": -2.2648820877075195,
      "step": 133,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.299023709902371,
      "grad_norm": 1.82398521900177,
      "learning_rate": 0.00019465090514086448,
      "logits/chosen": -0.2924862504005432,
      "logits/rejected": -0.49178171157836914,
      "logps/chosen": -5.639602184295654,
      "logps/rejected": -44.825313568115234,
      "loss": 0.5453966856002808,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3345707952976227,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07429855316877365,
      "rewards/margins": 2.4938178062438965,
      "rewards/rejected": -2.4195189476013184,
      "step": 134,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.301255230125523,
      "grad_norm": 3.8286569118499756,
      "learning_rate": 0.00019453113935347983,
      "logits/chosen": -0.29531559348106384,
      "logits/rejected": -0.43998682498931885,
      "logps/chosen": -10.71696662902832,
      "logps/rejected": -31.618389129638672,
      "loss": 1.1089136600494385,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7216206789016724,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1611023098230362,
      "rewards/margins": 1.5489929914474487,
      "rewards/rejected": -1.7100951671600342,
      "step": 135,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.30348675034867506,
      "grad_norm": 2.3694629669189453,
      "learning_rate": 0.000194410085271976,
      "logits/chosen": -0.3209247291088104,
      "logits/rejected": -0.47681719064712524,
      "logps/chosen": -8.806313514709473,
      "logps/rejected": -42.08256912231445,
      "loss": 0.8723977208137512,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6001561880111694,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.08420376479625702,
      "rewards/margins": 2.307250499725342,
      "rewards/rejected": -2.3914542198181152,
      "step": 136,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.30571827057182704,
      "grad_norm": 4.322510719299316,
      "learning_rate": 0.00019428774454610843,
      "logits/chosen": -0.3234959542751312,
      "logits/rejected": -0.43433526158332825,
      "logps/chosen": -8.827983856201172,
      "logps/rejected": -38.56624984741211,
      "loss": 0.8363085985183716,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5304505825042725,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03617179021239281,
      "rewards/margins": 2.3936820030212402,
      "rewards/rejected": -2.429853677749634,
      "step": 137,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.3079497907949791,
      "grad_norm": 1.8575319051742554,
      "learning_rate": 0.00019416411884316722,
      "logits/chosen": -0.19573748111724854,
      "logits/rejected": -0.39096614718437195,
      "logps/chosen": -5.302525043487549,
      "logps/rejected": -39.366905212402344,
      "loss": 0.5817482471466064,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3171233534812927,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06202463060617447,
      "rewards/margins": 2.1402504444122314,
      "rewards/rejected": -2.07822585105896,
      "step": 138,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.3101813110181311,
      "grad_norm": 1.6324138641357422,
      "learning_rate": 0.00019403920984795454,
      "logits/chosen": -0.18411299586296082,
      "logits/rejected": -0.3911224901676178,
      "logps/chosen": -4.253074645996094,
      "logps/rejected": -37.889976501464844,
      "loss": 0.6420720815658569,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3620051145553589,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1460508555173874,
      "rewards/margins": 2.2889294624328613,
      "rewards/rejected": -2.142878770828247,
      "step": 139,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.3124128312412831,
      "grad_norm": 2.665651798248291,
      "learning_rate": 0.00019391301926276156,
      "logits/chosen": -0.22396986186504364,
      "logits/rejected": -0.3946935534477234,
      "logps/chosen": -4.665857315063477,
      "logps/rejected": -40.48711395263672,
      "loss": 0.7322566509246826,
      "memory(GiB)": 46.81,
      "nll_loss": 0.41601666808128357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.039280496537685394,
      "rewards/margins": 2.420466899871826,
      "rewards/rejected": -2.459747314453125,
      "step": 140,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.31464435146443515,
      "grad_norm": 2.0447568893432617,
      "learning_rate": 0.00019378554880734527,
      "logits/chosen": -0.22006088495254517,
      "logits/rejected": -0.3875121772289276,
      "logps/chosen": -5.640967845916748,
      "logps/rejected": -47.2126579284668,
      "loss": 0.6299532651901245,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4075142443180084,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07418311387300491,
      "rewards/margins": 3.378464460372925,
      "rewards/rejected": -3.304281711578369,
      "step": 141,
      "train_speed(iter/s)": 0.010695
    },
    {
      "epoch": 0.3168758716875872,
      "grad_norm": 3.150909185409546,
      "learning_rate": 0.00019365680021890506,
      "logits/chosen": -0.2578600347042084,
      "logits/rejected": -0.35254308581352234,
      "logps/chosen": -8.539968490600586,
      "logps/rejected": -34.56965637207031,
      "loss": 0.9595645070075989,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5309237241744995,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.1485617607831955,
      "rewards/margins": 1.7760200500488281,
      "rewards/rejected": -1.9245820045471191,
      "step": 142,
      "train_speed(iter/s)": 0.010696
    },
    {
      "epoch": 0.31910739191073917,
      "grad_norm": 3.2599754333496094,
      "learning_rate": 0.0001935267752520591,
      "logits/chosen": -0.3178224563598633,
      "logits/rejected": -0.43051257729530334,
      "logps/chosen": -9.458460807800293,
      "logps/rejected": -43.98479080200195,
      "loss": 0.8238604068756104,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5105634331703186,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.19997264444828033,
      "rewards/margins": 2.638275146484375,
      "rewards/rejected": -2.8382480144500732,
      "step": 143,
      "train_speed(iter/s)": 0.010696
    },
    {
      "epoch": 0.3213389121338912,
      "grad_norm": 7.584465503692627,
      "learning_rate": 0.00019339547567882024,
      "logits/chosen": -0.2016015499830246,
      "logits/rejected": -0.36200886964797974,
      "logps/chosen": -9.776619911193848,
      "logps/rejected": -48.03835678100586,
      "loss": 0.9604641199111938,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7163732647895813,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.24650944769382477,
      "rewards/margins": 3.3983724117279053,
      "rewards/rejected": -3.151862621307373,
      "step": 144,
      "train_speed(iter/s)": 0.010696
    },
    {
      "epoch": 0.32357043235704325,
      "grad_norm": 3.555295944213867,
      "learning_rate": 0.0001932629032885721,
      "logits/chosen": -0.26102834939956665,
      "logits/rejected": -0.3585357069969177,
      "logps/chosen": -12.133922576904297,
      "logps/rejected": -48.155155181884766,
      "loss": 0.9948782920837402,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6610950231552124,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.4001372456550598,
      "rewards/margins": 2.751622200012207,
      "rewards/rejected": -3.151759386062622,
      "step": 145,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.32580195258019523,
      "grad_norm": 2.2978155612945557,
      "learning_rate": 0.0001931290598880445,
      "logits/chosen": -0.20670001208782196,
      "logits/rejected": -0.38169875741004944,
      "logps/chosen": -6.4796600341796875,
      "logps/rejected": -41.426429748535156,
      "loss": 0.6576346158981323,
      "memory(GiB)": 46.81,
      "nll_loss": 0.42290252447128296,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.11042710393667221,
      "rewards/margins": 2.6782710552215576,
      "rewards/rejected": -2.5678436756134033,
      "step": 146,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.32803347280334727,
      "grad_norm": 2.1882224082946777,
      "learning_rate": 0.00019299394730128895,
      "logits/chosen": -0.20903249084949493,
      "logits/rejected": -0.2923564314842224,
      "logps/chosen": -12.293680191040039,
      "logps/rejected": -50.68069076538086,
      "loss": 0.8728819489479065,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5617231130599976,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08868011832237244,
      "rewards/margins": 3.250684976577759,
      "rewards/rejected": -3.339365005493164,
      "step": 147,
      "train_speed(iter/s)": 0.010696
    },
    {
      "epoch": 0.3302649930264993,
      "grad_norm": 2.5009729862213135,
      "learning_rate": 0.00019285756736965367,
      "logits/chosen": -0.2492772340774536,
      "logits/rejected": -0.3531278669834137,
      "logps/chosen": -7.951318264007568,
      "logps/rejected": -34.95103073120117,
      "loss": 0.9188458323478699,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5677845478057861,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.015583137050271034,
      "rewards/margins": 2.127014398574829,
      "rewards/rejected": -2.111431360244751,
      "step": 148,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.33249651324965135,
      "grad_norm": 2.460752487182617,
      "learning_rate": 0.00019271992195175875,
      "logits/chosen": -0.31061843037605286,
      "logits/rejected": -0.42540180683135986,
      "logps/chosen": -8.994503021240234,
      "logps/rejected": -38.728179931640625,
      "loss": 0.9080746173858643,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5827285051345825,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09331205487251282,
      "rewards/margins": 2.394498348236084,
      "rewards/rejected": -2.3011860847473145,
      "step": 149,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 1.7428467273712158,
      "learning_rate": 0.00019258101292347042,
      "logits/chosen": -0.19874219596385956,
      "logits/rejected": -0.36566510796546936,
      "logps/chosen": -8.282856941223145,
      "logps/rejected": -47.80164337158203,
      "loss": 0.6850791573524475,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5025182366371155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09450387954711914,
      "rewards/margins": 2.836395502090454,
      "rewards/rejected": -2.741891622543335,
      "step": 150,
      "train_speed(iter/s)": 0.010697
    },
    {
      "epoch": 0.33695955369595537,
      "grad_norm": 1.7387069463729858,
      "learning_rate": 0.00019244084217787588,
      "logits/chosen": -0.22253645956516266,
      "logits/rejected": -0.36506688594818115,
      "logps/chosen": -8.663468360900879,
      "logps/rejected": -48.536319732666016,
      "loss": 0.7012121081352234,
      "memory(GiB)": 46.81,
      "nll_loss": 0.469788521528244,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1591811329126358,
      "rewards/margins": 2.9181699752807617,
      "rewards/rejected": -2.758988857269287,
      "step": 151,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.3391910739191074,
      "grad_norm": 2.085836887359619,
      "learning_rate": 0.00019229941162525726,
      "logits/chosen": -0.25783437490463257,
      "logits/rejected": -0.2909075915813446,
      "logps/chosen": -9.45044994354248,
      "logps/rejected": -33.906532287597656,
      "loss": 0.8592720627784729,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4804239869117737,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.12236574292182922,
      "rewards/margins": 1.7898168563842773,
      "rewards/rejected": -1.9121828079223633,
      "step": 152,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.3414225941422594,
      "grad_norm": 1.8399463891983032,
      "learning_rate": 0.00019215672319306565,
      "logits/chosen": -0.27110740542411804,
      "logits/rejected": -0.3448094129562378,
      "logps/chosen": -6.487441062927246,
      "logps/rejected": -33.882781982421875,
      "loss": 0.6851896047592163,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3750152587890625,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0861067920923233,
      "rewards/margins": 2.013005018234253,
      "rewards/rejected": -1.9268981218338013,
      "step": 153,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.34365411436541143,
      "grad_norm": 1.2756677865982056,
      "learning_rate": 0.00019201277882589487,
      "logits/chosen": -0.23933258652687073,
      "logits/rejected": -0.40502744913101196,
      "logps/chosen": -4.149328231811523,
      "logps/rejected": -56.27098083496094,
      "loss": 0.3939424157142639,
      "memory(GiB)": 46.81,
      "nll_loss": 0.240768164396286,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10984864085912704,
      "rewards/margins": 3.799041271209717,
      "rewards/rejected": -3.689192771911621,
      "step": 154,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.3458856345885635,
      "grad_norm": 2.6633708477020264,
      "learning_rate": 0.00019186758048545498,
      "logits/chosen": -0.2815844714641571,
      "logits/rejected": -0.422237366437912,
      "logps/chosen": -9.068172454833984,
      "logps/rejected": -44.39888000488281,
      "loss": 0.833120584487915,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5922810435295105,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.021779753267765045,
      "rewards/margins": 2.4418890476226807,
      "rewards/rejected": -2.4636685848236084,
      "step": 155,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.34811715481171546,
      "grad_norm": 3.4869775772094727,
      "learning_rate": 0.00019172113015054532,
      "logits/chosen": -0.38594380021095276,
      "logits/rejected": -0.4550650715827942,
      "logps/chosen": -9.38473129272461,
      "logps/rejected": -38.14994812011719,
      "loss": 0.6951225996017456,
      "memory(GiB)": 46.81,
      "nll_loss": 0.44333207607269287,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07232007384300232,
      "rewards/margins": 2.3049843311309814,
      "rewards/rejected": -2.2326643466949463,
      "step": 156,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.3503486750348675,
      "grad_norm": 2.0695393085479736,
      "learning_rate": 0.00019157342981702792,
      "logits/chosen": -0.30112287402153015,
      "logits/rejected": -0.4652690887451172,
      "logps/chosen": -4.151251792907715,
      "logps/rejected": -40.637821197509766,
      "loss": 0.5484941005706787,
      "memory(GiB)": 46.81,
      "nll_loss": 0.34684789180755615,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.14294274151325226,
      "rewards/margins": 2.3962206840515137,
      "rewards/rejected": -2.2532777786254883,
      "step": 157,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.35258019525801954,
      "grad_norm": 2.7222163677215576,
      "learning_rate": 0.0001914244814978001,
      "logits/chosen": -0.3103567063808441,
      "logits/rejected": -0.430208683013916,
      "logps/chosen": -12.615564346313477,
      "logps/rejected": -38.72933578491211,
      "loss": 0.793499231338501,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5542211532592773,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.015316572040319443,
      "rewards/margins": 2.3503148555755615,
      "rewards/rejected": -2.334998369216919,
      "step": 158,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.3548117154811716,
      "grad_norm": 2.9450817108154297,
      "learning_rate": 0.00019127428722276685,
      "logits/chosen": -0.24224838614463806,
      "logits/rejected": -0.43393394351005554,
      "logps/chosen": -8.04431438446045,
      "logps/rejected": -53.618770599365234,
      "loss": 0.753520131111145,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5266157984733582,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10486146807670593,
      "rewards/margins": 3.499842643737793,
      "rewards/rejected": -3.3949811458587646,
      "step": 159,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.35704323570432356,
      "grad_norm": 6.73776912689209,
      "learning_rate": 0.0001911228490388136,
      "logits/chosen": -0.3112480044364929,
      "logits/rejected": -0.4515516757965088,
      "logps/chosen": -9.544122695922852,
      "logps/rejected": -44.96870803833008,
      "loss": 0.7057361006736755,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5024272799491882,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22010847926139832,
      "rewards/margins": 2.9551522731781006,
      "rewards/rejected": -2.735043525695801,
      "step": 160,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.3592747559274756,
      "grad_norm": 3.6044814586639404,
      "learning_rate": 0.00019097016900977788,
      "logits/chosen": -0.3657134771347046,
      "logits/rejected": -0.45344415307044983,
      "logps/chosen": -11.24877643585205,
      "logps/rejected": -45.62288284301758,
      "loss": 0.5986043810844421,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3538128137588501,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06909701228141785,
      "rewards/margins": 3.109391927719116,
      "rewards/rejected": -3.040294885635376,
      "step": 161,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.36150627615062764,
      "grad_norm": 1.7556335926055908,
      "learning_rate": 0.00019081624921642157,
      "logits/chosen": -0.3330416977405548,
      "logits/rejected": -0.49046823382377625,
      "logps/chosen": -7.067329406738281,
      "logps/rejected": -46.18006896972656,
      "loss": 0.6361539363861084,
      "memory(GiB)": 46.81,
      "nll_loss": 0.39248889684677124,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14040817320346832,
      "rewards/margins": 2.906202793121338,
      "rewards/rejected": -2.765794515609741,
      "step": 162,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.3637377963737796,
      "grad_norm": 1.4350221157073975,
      "learning_rate": 0.00019066109175640224,
      "logits/chosen": -0.38426709175109863,
      "logits/rejected": -0.3983399271965027,
      "logps/chosen": -12.697317123413086,
      "logps/rejected": -50.39441680908203,
      "loss": 0.7568346858024597,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4696040451526642,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14664673805236816,
      "rewards/margins": 3.4247310161590576,
      "rewards/rejected": -3.2780845165252686,
      "step": 163,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.36596931659693166,
      "grad_norm": 2.5890817642211914,
      "learning_rate": 0.00019050469874424476,
      "logits/chosen": -0.32147228717803955,
      "logits/rejected": -0.45530298352241516,
      "logps/chosen": -8.81505012512207,
      "logps/rejected": -47.758445739746094,
      "loss": 0.8394909501075745,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5335902571678162,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.005741395056247711,
      "rewards/margins": 3.229443073272705,
      "rewards/rejected": -3.23518443107605,
      "step": 164,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.3682008368200837,
      "grad_norm": 2.527979850769043,
      "learning_rate": 0.0001903470723113124,
      "logits/chosen": -0.3687136173248291,
      "logits/rejected": -0.36498233675956726,
      "logps/chosen": -12.974212646484375,
      "logps/rejected": -32.33938217163086,
      "loss": 0.9435381889343262,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5042305588722229,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.09565389156341553,
      "rewards/margins": 1.7492725849151611,
      "rewards/rejected": -1.8449265956878662,
      "step": 165,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.3704323570432357,
      "grad_norm": 2.3836722373962402,
      "learning_rate": 0.00019018821460577777,
      "logits/chosen": -0.2881522476673126,
      "logits/rejected": -0.39948752522468567,
      "logps/chosen": -14.373290061950684,
      "logps/rejected": -46.974891662597656,
      "loss": 1.0135047435760498,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6972138285636902,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.46250468492507935,
      "rewards/margins": 2.5046420097351074,
      "rewards/rejected": -2.967146873474121,
      "step": 166,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.3726638772663877,
      "grad_norm": 2.324857234954834,
      "learning_rate": 0.00019002812779259364,
      "logits/chosen": -0.29876166582107544,
      "logits/rejected": -0.47440457344055176,
      "logps/chosen": -6.713703632354736,
      "logps/rejected": -38.95991134643555,
      "loss": 0.7639641761779785,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4946966767311096,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.18299029767513275,
      "rewards/margins": 2.441457748413086,
      "rewards/rejected": -2.258467435836792,
      "step": 167,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.37489539748953976,
      "grad_norm": 5.72042179107666,
      "learning_rate": 0.00018986681405346322,
      "logits/chosen": -0.28162381052970886,
      "logits/rejected": -0.47184035181999207,
      "logps/chosen": -9.439067840576172,
      "logps/rejected": -53.3689079284668,
      "loss": 0.964309811592102,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6164814233779907,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.01046261191368103,
      "rewards/margins": 3.1293952465057373,
      "rewards/rejected": -3.1398582458496094,
      "step": 168,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.37712691771269174,
      "grad_norm": 4.976809978485107,
      "learning_rate": 0.00018970427558681082,
      "logits/chosen": -0.25434356927871704,
      "logits/rejected": -0.48102837800979614,
      "logps/chosen": -8.21963119506836,
      "logps/rejected": -65.61998748779297,
      "loss": 0.8252819776535034,
      "memory(GiB)": 46.81,
      "nll_loss": 0.646252453327179,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1419672667980194,
      "rewards/margins": 4.092907428741455,
      "rewards/rejected": -3.9509406089782715,
      "step": 169,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.3793584379358438,
      "grad_norm": 3.3867485523223877,
      "learning_rate": 0.0001895405146077514,
      "logits/chosen": -0.34642744064331055,
      "logits/rejected": -0.4897533357143402,
      "logps/chosen": -5.762866020202637,
      "logps/rejected": -46.86370086669922,
      "loss": 0.7062004804611206,
      "memory(GiB)": 46.81,
      "nll_loss": 0.33989417552948,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06110583245754242,
      "rewards/margins": 2.4862968921661377,
      "rewards/rejected": -2.5474026203155518,
      "step": 170,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.3815899581589958,
      "grad_norm": 1.8158555030822754,
      "learning_rate": 0.00018937553334806077,
      "logits/chosen": -0.2750059962272644,
      "logits/rejected": -0.4136987030506134,
      "logps/chosen": -8.728262901306152,
      "logps/rejected": -45.89287185668945,
      "loss": 0.7403351068496704,
      "memory(GiB)": 46.81,
      "nll_loss": 0.49930721521377563,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16491073369979858,
      "rewards/margins": 3.0132622718811035,
      "rewards/rejected": -2.8483517169952393,
      "step": 171,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.38382147838214786,
      "grad_norm": 1.472412109375,
      "learning_rate": 0.000189209334056145,
      "logits/chosen": -0.27084222435951233,
      "logits/rejected": -0.434164434671402,
      "logps/chosen": -5.168161392211914,
      "logps/rejected": -45.41401672363281,
      "loss": 0.6193027496337891,
      "memory(GiB)": 46.81,
      "nll_loss": 0.42660266160964966,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.12104358524084091,
      "rewards/margins": 2.716996431350708,
      "rewards/rejected": -2.5959529876708984,
      "step": 172,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.38605299860529985,
      "grad_norm": 3.4075303077697754,
      "learning_rate": 0.00018904191899700977,
      "logits/chosen": -0.3655165433883667,
      "logits/rejected": -0.41131308674812317,
      "logps/chosen": -6.5695905685424805,
      "logps/rejected": -32.802406311035156,
      "loss": 0.7823173403739929,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4896646738052368,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10015285015106201,
      "rewards/margins": 1.8082492351531982,
      "rewards/rejected": -1.7080962657928467,
      "step": 173,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.3882845188284519,
      "grad_norm": 1.7677769660949707,
      "learning_rate": 0.0001888732904522296,
      "logits/chosen": -0.3326292634010315,
      "logits/rejected": -0.4164149761199951,
      "logps/chosen": -5.277183532714844,
      "logps/rejected": -42.28554916381836,
      "loss": 0.6526207327842712,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4348045885562897,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10670876502990723,
      "rewards/margins": 2.7690269947052,
      "rewards/rejected": -2.662318468093872,
      "step": 174,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.3905160390516039,
      "grad_norm": 2.253504753112793,
      "learning_rate": 0.00018870345071991663,
      "logits/chosen": -0.3795526325702667,
      "logits/rejected": -0.4494696855545044,
      "logps/chosen": -4.6198410987854,
      "logps/rejected": -34.95786666870117,
      "loss": 0.5223369598388672,
      "memory(GiB)": 46.81,
      "nll_loss": 0.29292282462120056,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05781913921236992,
      "rewards/margins": 2.233022928237915,
      "rewards/rejected": -2.175203800201416,
      "step": 175,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.3927475592747559,
      "grad_norm": 4.090317726135254,
      "learning_rate": 0.00018853240211468944,
      "logits/chosen": -0.33655816316604614,
      "logits/rejected": -0.3891099691390991,
      "logps/chosen": -12.897551536560059,
      "logps/rejected": -34.91562271118164,
      "loss": 0.9967415928840637,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5328224301338196,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0005232337862253189,
      "rewards/margins": 1.7757031917572021,
      "rewards/rejected": -1.7762264013290405,
      "step": 176,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.39497907949790795,
      "grad_norm": 1.5479252338409424,
      "learning_rate": 0.0001883601469676414,
      "logits/chosen": -0.31817540526390076,
      "logits/rejected": -0.4586165249347687,
      "logps/chosen": -4.228600025177002,
      "logps/rejected": -44.335289001464844,
      "loss": 0.5654776096343994,
      "memory(GiB)": 46.81,
      "nll_loss": 0.354885458946228,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04608811438083649,
      "rewards/margins": 2.883877754211426,
      "rewards/rejected": -2.83778977394104,
      "step": 177,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.39721059972106,
      "grad_norm": 1.458658218383789,
      "learning_rate": 0.00018818668762630888,
      "logits/chosen": -0.3560311794281006,
      "logits/rejected": -0.46038633584976196,
      "logps/chosen": -6.351040363311768,
      "logps/rejected": -38.50391387939453,
      "loss": 0.5693007111549377,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3589138686656952,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06642849743366241,
      "rewards/margins": 2.5289833545684814,
      "rewards/rejected": -2.462554693222046,
      "step": 178,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.39944211994421197,
      "grad_norm": 1.4788439273834229,
      "learning_rate": 0.00018801202645463947,
      "logits/chosen": -0.44740062952041626,
      "logits/rejected": -0.48686325550079346,
      "logps/chosen": -7.422541618347168,
      "logps/rejected": -34.7234001159668,
      "loss": 0.7467449903488159,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5351516008377075,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2607961595058441,
      "rewards/margins": 2.635673999786377,
      "rewards/rejected": -2.374877691268921,
      "step": 179,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.401673640167364,
      "grad_norm": 1.2784523963928223,
      "learning_rate": 0.00018783616583295943,
      "logits/chosen": -0.34507742524147034,
      "logits/rejected": -0.46668770909309387,
      "logps/chosen": -4.764893531799316,
      "logps/rejected": -44.04815673828125,
      "loss": 0.49975255131721497,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3011859357357025,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.152861088514328,
      "rewards/margins": 2.924600124359131,
      "rewards/rejected": -2.7717387676239014,
      "step": 180,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.40390516039051605,
      "grad_norm": 1.7386562824249268,
      "learning_rate": 0.0001876591081579415,
      "logits/chosen": -0.38848260045051575,
      "logits/rejected": -0.48753347992897034,
      "logps/chosen": -5.946084499359131,
      "logps/rejected": -40.89453125,
      "loss": 0.59507155418396,
      "memory(GiB)": 46.81,
      "nll_loss": 0.37911903858184814,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1187826544046402,
      "rewards/margins": 2.863964319229126,
      "rewards/rejected": -2.7451813220977783,
      "step": 181,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.4061366806136681,
      "grad_norm": 1.747938632965088,
      "learning_rate": 0.0001874808558425722,
      "logits/chosen": -0.3996775150299072,
      "logits/rejected": -0.4858594536781311,
      "logps/chosen": -8.173686027526855,
      "logps/rejected": -36.68798065185547,
      "loss": 0.7136989831924438,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4385717213153839,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21724800765514374,
      "rewards/margins": 2.21327543258667,
      "rewards/rejected": -1.9960275888442993,
      "step": 182,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.40836820083682007,
      "grad_norm": 2.1921677589416504,
      "learning_rate": 0.00018730141131611882,
      "logits/chosen": -0.3768147826194763,
      "logits/rejected": -0.4843496084213257,
      "logps/chosen": -7.464463233947754,
      "logps/rejected": -37.9464111328125,
      "loss": 0.744111180305481,
      "memory(GiB)": 46.81,
      "nll_loss": 0.43925032019615173,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.062185730785131454,
      "rewards/margins": 2.2055001258850098,
      "rewards/rejected": -2.1433143615722656,
      "step": 183,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.4105997210599721,
      "grad_norm": 3.3944175243377686,
      "learning_rate": 0.0001871207770240965,
      "logits/chosen": -0.3745571970939636,
      "logits/rejected": -0.5446013808250427,
      "logps/chosen": -6.670903205871582,
      "logps/rejected": -52.90716552734375,
      "loss": 0.8081664443016052,
      "memory(GiB)": 46.81,
      "nll_loss": 0.546490490436554,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06610552966594696,
      "rewards/margins": 3.103334426879883,
      "rewards/rejected": -3.037229061126709,
      "step": 184,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.41283124128312415,
      "grad_norm": 1.7184181213378906,
      "learning_rate": 0.00018693895542823476,
      "logits/chosen": -0.42309850454330444,
      "logits/rejected": -0.5455373525619507,
      "logps/chosen": -4.061583518981934,
      "logps/rejected": -46.45550537109375,
      "loss": 0.5212042331695557,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3185817003250122,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06332439184188843,
      "rewards/margins": 3.2307968139648438,
      "rewards/rejected": -3.1674723625183105,
      "step": 185,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.41506276150627613,
      "grad_norm": 2.1810686588287354,
      "learning_rate": 0.00018675594900644394,
      "logits/chosen": -0.43507513403892517,
      "logits/rejected": -0.5661131739616394,
      "logps/chosen": -7.813204765319824,
      "logps/rejected": -47.85923385620117,
      "loss": 0.6977582573890686,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4816473722457886,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.05514959990978241,
      "rewards/margins": 3.3984975814819336,
      "rewards/rejected": -3.3433480262756348,
      "step": 186,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.4172942817294282,
      "grad_norm": 8.446174621582031,
      "learning_rate": 0.0001865717602527816,
      "logits/chosen": -0.3857502043247223,
      "logits/rejected": -0.5707287788391113,
      "logps/chosen": -5.875633239746094,
      "logps/rejected": -52.29627990722656,
      "loss": 0.8916729688644409,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6320705413818359,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10125800967216492,
      "rewards/margins": 3.7010817527770996,
      "rewards/rejected": -3.599823474884033,
      "step": 187,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.4195258019525802,
      "grad_norm": 1.2875564098358154,
      "learning_rate": 0.00018638639167741834,
      "logits/chosen": -0.45664966106414795,
      "logits/rejected": -0.6208757162094116,
      "logps/chosen": -6.463584899902344,
      "logps/rejected": -63.02524185180664,
      "loss": 0.5230106115341187,
      "memory(GiB)": 46.81,
      "nll_loss": 0.38262802362442017,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.25487378239631653,
      "rewards/margins": 4.994178295135498,
      "rewards/rejected": -4.739304065704346,
      "step": 188,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.4217573221757322,
      "grad_norm": 2.2951433658599854,
      "learning_rate": 0.00018619984580660367,
      "logits/chosen": -0.4468421936035156,
      "logits/rejected": -0.590311586856842,
      "logps/chosen": -7.629635334014893,
      "logps/rejected": -60.66893768310547,
      "loss": 0.6314592361450195,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4014724791049957,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.0873238816857338,
      "rewards/margins": 4.23132848739624,
      "rewards/rejected": -4.318652629852295,
      "step": 189,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.42398884239888424,
      "grad_norm": 3.432699680328369,
      "learning_rate": 0.00018601212518263156,
      "logits/chosen": -0.4163021147251129,
      "logits/rejected": -0.5166957378387451,
      "logps/chosen": -10.489439010620117,
      "logps/rejected": -59.12989044189453,
      "loss": 0.9744394421577454,
      "memory(GiB)": 46.81,
      "nll_loss": 0.7091968655586243,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.017084941267967224,
      "rewards/margins": 4.347151279449463,
      "rewards/rejected": -4.330066680908203,
      "step": 190,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.4262203626220363,
      "grad_norm": 3.6154932975769043,
      "learning_rate": 0.00018582323236380588,
      "logits/chosen": -0.5206790566444397,
      "logits/rejected": -0.5340564250946045,
      "logps/chosen": -14.040782928466797,
      "logps/rejected": -46.77815628051758,
      "loss": 1.0751701593399048,
      "memory(GiB)": 46.81,
      "nll_loss": 0.665191113948822,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.24230876564979553,
      "rewards/margins": 3.153132438659668,
      "rewards/rejected": -3.3954415321350098,
      "step": 191,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.42845188284518826,
      "grad_norm": 8.427726745605469,
      "learning_rate": 0.00018563316992440543,
      "logits/chosen": -0.4730026423931122,
      "logits/rejected": -0.6191343069076538,
      "logps/chosen": -6.7660651206970215,
      "logps/rejected": -55.98576354980469,
      "loss": 0.8084225058555603,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5363692045211792,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10752980411052704,
      "rewards/margins": 3.833192825317383,
      "rewards/rejected": -3.940722703933716,
      "step": 192,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.4306834030683403,
      "grad_norm": 2.5356807708740234,
      "learning_rate": 0.00018544194045464886,
      "logits/chosen": -0.41117027401924133,
      "logits/rejected": -0.5410706400871277,
      "logps/chosen": -6.973834991455078,
      "logps/rejected": -47.055511474609375,
      "loss": 0.7811009883880615,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5517580509185791,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13715188205242157,
      "rewards/margins": 3.0469465255737305,
      "rewards/rejected": -2.909794330596924,
      "step": 193,
      "train_speed(iter/s)": 0.010692
    },
    {
      "epoch": 0.43291492329149234,
      "grad_norm": 1.7070624828338623,
      "learning_rate": 0.00018524954656065946,
      "logits/chosen": -0.46515780687332153,
      "logits/rejected": -0.5741381645202637,
      "logps/chosen": -7.333560943603516,
      "logps/rejected": -46.890380859375,
      "loss": 0.46373608708381653,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3009029030799866,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09283138811588287,
      "rewards/margins": 3.2314822673797607,
      "rewards/rejected": -3.138651132583618,
      "step": 194,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.4351464435146444,
      "grad_norm": 2.165066957473755,
      "learning_rate": 0.00018505599086442956,
      "logits/chosen": -0.4237922728061676,
      "logits/rejected": -0.6133349537849426,
      "logps/chosen": -2.157940626144409,
      "logps/rejected": -44.79521942138672,
      "loss": 0.48500534892082214,
      "memory(GiB)": 46.81,
      "nll_loss": 0.28576743602752686,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1598954200744629,
      "rewards/margins": 2.9778366088867188,
      "rewards/rejected": -2.817941188812256,
      "step": 195,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.43737796373779636,
      "grad_norm": 2.6087758541107178,
      "learning_rate": 0.0001848612760037848,
      "logits/chosen": -0.42983922362327576,
      "logits/rejected": -0.5865707397460938,
      "logps/chosen": -4.183155536651611,
      "logps/rejected": -44.91338348388672,
      "loss": 0.4380960762500763,
      "memory(GiB)": 46.81,
      "nll_loss": 0.22082072496414185,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1885657161474228,
      "rewards/margins": 2.626762628555298,
      "rewards/rejected": -2.438196897506714,
      "step": 196,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.4396094839609484,
      "grad_norm": 2.3417139053344727,
      "learning_rate": 0.0001846654046323482,
      "logits/chosen": -0.4529270827770233,
      "logits/rejected": -0.5250231027603149,
      "logps/chosen": -10.214118957519531,
      "logps/rejected": -33.07242202758789,
      "loss": 0.9449623823165894,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6152971982955933,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03806466981768608,
      "rewards/margins": 1.6490262746810913,
      "rewards/rejected": -1.6870909929275513,
      "step": 197,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.44184100418410044,
      "grad_norm": 2.435980796813965,
      "learning_rate": 0.0001844683794195041,
      "logits/chosen": -0.41686204075813293,
      "logits/rejected": -0.5872527956962585,
      "logps/chosen": -5.5450968742370605,
      "logps/rejected": -41.29132843017578,
      "loss": 0.6443852782249451,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3510324954986572,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10375159978866577,
      "rewards/margins": 2.4684395790100098,
      "rewards/rejected": -2.364687919616699,
      "step": 198,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.4440725244072524,
      "grad_norm": 2.279975414276123,
      "learning_rate": 0.00018427020305036157,
      "logits/chosen": -0.4846893548965454,
      "logits/rejected": -0.5544926524162292,
      "logps/chosen": -5.250210762023926,
      "logps/rejected": -37.56634521484375,
      "loss": 0.6850971579551697,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3965678811073303,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.009095368906855583,
      "rewards/margins": 2.2770018577575684,
      "rewards/rejected": -2.267906427383423,
      "step": 199,
      "train_speed(iter/s)": 0.010693
    },
    {
      "epoch": 0.44630404463040446,
      "grad_norm": 3.310929298400879,
      "learning_rate": 0.00018407087822571794,
      "logits/chosen": -0.45392054319381714,
      "logits/rejected": -0.5642043352127075,
      "logps/chosen": -9.782092094421387,
      "logps/rejected": -40.09915542602539,
      "loss": 0.8508836030960083,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6215177178382874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1146484911441803,
      "rewards/margins": 2.442267656326294,
      "rewards/rejected": -2.3276190757751465,
      "step": 200,
      "train_speed(iter/s)": 0.010694
    },
    {
      "epoch": 0.4485355648535565,
      "grad_norm": 2.2623071670532227,
      "learning_rate": 0.00018387040766202202,
      "logits/chosen": -0.45955681800842285,
      "logits/rejected": -0.5978164672851562,
      "logps/chosen": -9.393783569335938,
      "logps/rejected": -52.4150390625,
      "loss": 0.6487902998924255,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4702387750148773,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14986836910247803,
      "rewards/margins": 3.318939208984375,
      "rewards/rejected": -3.1690707206726074,
      "step": 201,
      "train_speed(iter/s)": 0.010683
    },
    {
      "epoch": 0.4507670850767085,
      "grad_norm": 1.4719666242599487,
      "learning_rate": 0.00018366879409133702,
      "logits/chosen": -0.5009268522262573,
      "logits/rejected": -0.625489354133606,
      "logps/chosen": -7.751692771911621,
      "logps/rejected": -50.78370666503906,
      "loss": 0.5859130620956421,
      "memory(GiB)": 46.81,
      "nll_loss": 0.39871278405189514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10605036467313766,
      "rewards/margins": 3.2280044555664062,
      "rewards/rejected": -3.1219542026519775,
      "step": 202,
      "train_speed(iter/s)": 0.010683
    },
    {
      "epoch": 0.4529986052998605,
      "grad_norm": 3.2725584506988525,
      "learning_rate": 0.00018346604026130335,
      "logits/chosen": -0.5253587961196899,
      "logits/rejected": -0.6153156161308289,
      "logps/chosen": -7.448980331420898,
      "logps/rejected": -37.25505828857422,
      "loss": 0.8574545383453369,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5657089352607727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08424842357635498,
      "rewards/margins": 2.4470019340515137,
      "rewards/rejected": -2.3627536296844482,
      "step": 203,
      "train_speed(iter/s)": 0.010683
    },
    {
      "epoch": 0.45523012552301256,
      "grad_norm": 1.6742641925811768,
      "learning_rate": 0.00018326214893510113,
      "logits/chosen": -0.5298970937728882,
      "logits/rejected": -0.5919500589370728,
      "logps/chosen": -6.125566005706787,
      "logps/rejected": -37.60496139526367,
      "loss": 0.637344479560852,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3929066061973572,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13108696043491364,
      "rewards/margins": 2.768951654434204,
      "rewards/rejected": -2.637864589691162,
      "step": 204,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.45746164574616455,
      "grad_norm": 1.8297412395477295,
      "learning_rate": 0.00018305712289141263,
      "logits/chosen": -0.5556704998016357,
      "logits/rejected": -0.6641337871551514,
      "logps/chosen": -7.0842132568359375,
      "logps/rejected": -43.63640213012695,
      "loss": 0.611258327960968,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3571435213088989,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08243419229984283,
      "rewards/margins": 2.901592969894409,
      "rewards/rejected": -2.8191587924957275,
      "step": 205,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.4596931659693166,
      "grad_norm": 2.103322982788086,
      "learning_rate": 0.00018285096492438424,
      "logits/chosen": -0.5411264896392822,
      "logits/rejected": -0.6788846254348755,
      "logps/chosen": -7.095435619354248,
      "logps/rejected": -55.614654541015625,
      "loss": 0.5769423842430115,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3610311448574066,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.028350576758384705,
      "rewards/margins": 3.771873950958252,
      "rewards/rejected": -3.743523120880127,
      "step": 206,
      "train_speed(iter/s)": 0.010684
    },
    {
      "epoch": 0.4619246861924686,
      "grad_norm": 5.2644124031066895,
      "learning_rate": 0.00018264367784358854,
      "logits/chosen": -0.5137989521026611,
      "logits/rejected": -0.6350421905517578,
      "logps/chosen": -8.56877326965332,
      "logps/rejected": -47.056922912597656,
      "loss": 0.8458958268165588,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6215453743934631,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1831740289926529,
      "rewards/margins": 3.3318536281585693,
      "rewards/rejected": -3.148679494857788,
      "step": 207,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.46415620641562066,
      "grad_norm": 2.3040452003479004,
      "learning_rate": 0.00018243526447398595,
      "logits/chosen": -0.5189839005470276,
      "logits/rejected": -0.6215395927429199,
      "logps/chosen": -8.22704792022705,
      "logps/rejected": -45.81852340698242,
      "loss": 0.6660462617874146,
      "memory(GiB)": 46.81,
      "nll_loss": 0.42924296855926514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.015878167003393173,
      "rewards/margins": 3.395660877227783,
      "rewards/rejected": -3.3797831535339355,
      "step": 208,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.46638772663877265,
      "grad_norm": 4.965414047241211,
      "learning_rate": 0.00018222572765588625,
      "logits/chosen": -0.4948608875274658,
      "logits/rejected": -0.5670303106307983,
      "logps/chosen": -14.887557983398438,
      "logps/rejected": -48.515113830566406,
      "loss": 0.8501808643341064,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5177828669548035,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.011799663305282593,
      "rewards/margins": 3.1876440048217773,
      "rewards/rejected": -3.175844192504883,
      "step": 209,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4686192468619247,
      "grad_norm": 2.125880479812622,
      "learning_rate": 0.00018201507024490988,
      "logits/chosen": -0.4645600914955139,
      "logits/rejected": -0.5849769711494446,
      "logps/chosen": -14.342391967773438,
      "logps/rejected": -53.85160446166992,
      "loss": 0.7610176205635071,
      "memory(GiB)": 46.81,
      "nll_loss": 0.60414057970047,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1484633982181549,
      "rewards/margins": 3.265408992767334,
      "rewards/rejected": -3.11694598197937,
      "step": 210,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4708507670850767,
      "grad_norm": 1.3584539890289307,
      "learning_rate": 0.0001818032951119489,
      "logits/chosen": -0.4379713833332062,
      "logits/rejected": -0.6403225660324097,
      "logps/chosen": -3.4099626541137695,
      "logps/rejected": -55.08962631225586,
      "loss": 0.4291754364967346,
      "memory(GiB)": 46.81,
      "nll_loss": 0.28185123205184937,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2514503002166748,
      "rewards/margins": 3.88580584526062,
      "rewards/rejected": -3.6343557834625244,
      "step": 211,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4730822873082287,
      "grad_norm": 1.7748886346817017,
      "learning_rate": 0.00018159040514312803,
      "logits/chosen": -0.5069133043289185,
      "logits/rejected": -0.5805174708366394,
      "logps/chosen": -8.58336067199707,
      "logps/rejected": -35.97822189331055,
      "loss": 0.663266658782959,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3847760558128357,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2307383120059967,
      "rewards/margins": 2.5746662616729736,
      "rewards/rejected": -2.343928098678589,
      "step": 212,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.47531380753138075,
      "grad_norm": 1.843264102935791,
      "learning_rate": 0.00018137640323976536,
      "logits/chosen": -0.4634440243244171,
      "logits/rejected": -0.5595231056213379,
      "logps/chosen": -12.369077682495117,
      "logps/rejected": -49.64701843261719,
      "loss": 0.7064059972763062,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5452557802200317,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06534570455551147,
      "rewards/margins": 3.548327684402466,
      "rewards/rejected": -3.4829816818237305,
      "step": 213,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4775453277545328,
      "grad_norm": 3.4433913230895996,
      "learning_rate": 0.00018116129231833249,
      "logits/chosen": -0.4008879065513611,
      "logits/rejected": -0.5341296195983887,
      "logps/chosen": -8.161286354064941,
      "logps/rejected": -41.809959411621094,
      "loss": 0.7387677431106567,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4233713746070862,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04444505646824837,
      "rewards/margins": 2.688593626022339,
      "rewards/rejected": -2.644148588180542,
      "step": 214,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4797768479776848,
      "grad_norm": 2.6310174465179443,
      "learning_rate": 0.00018094507531041517,
      "logits/chosen": -0.4581325352191925,
      "logits/rejected": -0.5395053625106812,
      "logps/chosen": -12.050057411193848,
      "logps/rejected": -46.67640686035156,
      "loss": 0.6482417583465576,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4028882384300232,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.03946290165185928,
      "rewards/margins": 3.062953233718872,
      "rewards/rejected": -3.0234906673431396,
      "step": 215,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4820083682008368,
      "grad_norm": 2.039609432220459,
      "learning_rate": 0.00018072775516267306,
      "logits/chosen": -0.40940549969673157,
      "logits/rejected": -0.5348517894744873,
      "logps/chosen": -5.345945835113525,
      "logps/rejected": -55.2205696105957,
      "loss": 0.588547945022583,
      "memory(GiB)": 46.81,
      "nll_loss": 0.35309699177742004,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.013576401397585869,
      "rewards/margins": 3.3509364128112793,
      "rewards/rejected": -3.33735990524292,
      "step": 216,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.48423988842398885,
      "grad_norm": 1.844130039215088,
      "learning_rate": 0.00018050933483679976,
      "logits/chosen": -0.388172447681427,
      "logits/rejected": -0.5932977199554443,
      "logps/chosen": -7.16074800491333,
      "logps/rejected": -57.54563903808594,
      "loss": 0.5982820987701416,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3665384352207184,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12162206321954727,
      "rewards/margins": 3.5915048122406006,
      "rewards/rejected": -3.4698829650878906,
      "step": 217,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4864714086471409,
      "grad_norm": 3.176985263824463,
      "learning_rate": 0.0001802898173094824,
      "logits/chosen": -0.4476504623889923,
      "logits/rejected": -0.6101452112197876,
      "logps/chosen": -8.568498611450195,
      "logps/rejected": -55.76399612426758,
      "loss": 0.759227454662323,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5219362378120422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05749676004052162,
      "rewards/margins": 4.200704574584961,
      "rewards/rejected": -4.143207550048828,
      "step": 218,
      "train_speed(iter/s)": 0.010685
    },
    {
      "epoch": 0.4887029288702929,
      "grad_norm": 8.979230880737305,
      "learning_rate": 0.0001800692055723609,
      "logits/chosen": -0.5705654621124268,
      "logits/rejected": -0.6717063188552856,
      "logps/chosen": -9.38418197631836,
      "logps/rejected": -48.490394592285156,
      "loss": 0.9973833560943604,
      "memory(GiB)": 46.81,
      "nll_loss": 0.765608549118042,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.36961740255355835,
      "rewards/margins": 3.4103264808654785,
      "rewards/rejected": -3.7799437046051025,
      "step": 219,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.4909344490934449,
      "grad_norm": 13.02912425994873,
      "learning_rate": 0.0001798475026319875,
      "logits/chosen": -0.48003196716308594,
      "logits/rejected": -0.6781996488571167,
      "logps/chosen": -7.269015789031982,
      "logps/rejected": -62.30265808105469,
      "loss": 0.7909015417098999,
      "memory(GiB)": 46.81,
      "nll_loss": 0.53118896484375,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.15388834476470947,
      "rewards/margins": 4.509274959564209,
      "rewards/rejected": -4.663163185119629,
      "step": 220,
      "train_speed(iter/s)": 0.010686
    },
    {
      "epoch": 0.49316596931659695,
      "grad_norm": 6.664190292358398,
      "learning_rate": 0.00017962471150978563,
      "logits/chosen": -0.5324437022209167,
      "logits/rejected": -0.6654635667800903,
      "logps/chosen": -7.194028377532959,
      "logps/rejected": -59.914695739746094,
      "loss": 0.8783823251724243,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6288973689079285,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.018478773534297943,
      "rewards/margins": 4.640980243682861,
      "rewards/rejected": -4.622502326965332,
      "step": 221,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.49539748953974894,
      "grad_norm": 2.010087251663208,
      "learning_rate": 0.00017940083524200858,
      "logits/chosen": -0.43658989667892456,
      "logits/rejected": -0.5585737228393555,
      "logps/chosen": -8.627237319946289,
      "logps/rejected": -48.756126403808594,
      "loss": 0.5573357939720154,
      "memory(GiB)": 46.81,
      "nll_loss": 0.39346298575401306,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.34565243124961853,
      "rewards/margins": 3.3448474407196045,
      "rewards/rejected": -2.999195098876953,
      "step": 222,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.497629009762901,
      "grad_norm": 2.036358118057251,
      "learning_rate": 0.00017917587687969847,
      "logits/chosen": -0.434632807970047,
      "logits/rejected": -0.5482152104377747,
      "logps/chosen": -11.499567985534668,
      "logps/rejected": -41.673377990722656,
      "loss": 0.8241864442825317,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6470968723297119,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23597590625286102,
      "rewards/margins": 2.9532856941223145,
      "rewards/rejected": -2.7173097133636475,
      "step": 223,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.499860529986053,
      "grad_norm": 2.800816297531128,
      "learning_rate": 0.00017894983948864431,
      "logits/chosen": -0.42586764693260193,
      "logits/rejected": -0.5355687141418457,
      "logps/chosen": -8.567970275878906,
      "logps/rejected": -38.82350158691406,
      "loss": 0.8209130167961121,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5143148899078369,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0072904713451862335,
      "rewards/margins": 2.314070224761963,
      "rewards/rejected": -2.3213610649108887,
      "step": 224,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 3.365342140197754,
      "learning_rate": 0.0001787227261493405,
      "logits/chosen": -0.423734188079834,
      "logits/rejected": -0.568479061126709,
      "logps/chosen": -7.375796794891357,
      "logps/rejected": -49.31288146972656,
      "loss": 0.7071059942245483,
      "memory(GiB)": 46.81,
      "nll_loss": 0.46694087982177734,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.004096744582056999,
      "rewards/margins": 3.2242605686187744,
      "rewards/rejected": -3.2283570766448975,
      "step": 225,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.5043235704323571,
      "grad_norm": 1.2737809419631958,
      "learning_rate": 0.0001784945399569447,
      "logits/chosen": -0.36446237564086914,
      "logits/rejected": -0.5933336615562439,
      "logps/chosen": -3.509737730026245,
      "logps/rejected": -58.02750778198242,
      "loss": 0.33625951409339905,
      "memory(GiB)": 46.81,
      "nll_loss": 0.22468024492263794,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18253092467784882,
      "rewards/margins": 3.6850192546844482,
      "rewards/rejected": -3.502488136291504,
      "step": 226,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.5065550906555091,
      "grad_norm": 2.068448305130005,
      "learning_rate": 0.00017826528402123564,
      "logits/chosen": -0.42301344871520996,
      "logits/rejected": -0.5985809564590454,
      "logps/chosen": -2.5439209938049316,
      "logps/rejected": -45.29689025878906,
      "loss": 0.36974459886550903,
      "memory(GiB)": 46.81,
      "nll_loss": 0.231589674949646,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2267550826072693,
      "rewards/margins": 3.147808790206909,
      "rewards/rejected": -2.9210541248321533,
      "step": 227,
      "train_speed(iter/s)": 0.010687
    },
    {
      "epoch": 0.5087866108786611,
      "grad_norm": 1.3184460401535034,
      "learning_rate": 0.00017803496146657084,
      "logits/chosen": -0.4008505344390869,
      "logits/rejected": -0.592756450176239,
      "logps/chosen": -6.708431243896484,
      "logps/rejected": -48.843502044677734,
      "loss": 0.6300328969955444,
      "memory(GiB)": 46.81,
      "nll_loss": 0.46876925230026245,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32074451446533203,
      "rewards/margins": 3.1505868434906006,
      "rewards/rejected": -2.8298423290252686,
      "step": 228,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.5110181311018132,
      "grad_norm": 2.306436538696289,
      "learning_rate": 0.00017780357543184397,
      "logits/chosen": -0.4543289244174957,
      "logits/rejected": -0.5702416300773621,
      "logps/chosen": -7.456311225891113,
      "logps/rejected": -55.43143081665039,
      "loss": 0.6797499060630798,
      "memory(GiB)": 46.81,
      "nll_loss": 0.43957531452178955,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.061677221208810806,
      "rewards/margins": 4.144193649291992,
      "rewards/rejected": -4.082516670227051,
      "step": 229,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.5132496513249651,
      "grad_norm": 5.570302963256836,
      "learning_rate": 0.000177571129070442,
      "logits/chosen": -0.5052129626274109,
      "logits/rejected": -0.5332801342010498,
      "logps/chosen": -10.987539291381836,
      "logps/rejected": -43.629920959472656,
      "loss": 0.8144917488098145,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5699902772903442,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.07897716760635376,
      "rewards/margins": 2.79160213470459,
      "rewards/rejected": -2.870579481124878,
      "step": 230,
      "train_speed(iter/s)": 0.010688
    },
    {
      "epoch": 0.5154811715481171,
      "grad_norm": 1.758008360862732,
      "learning_rate": 0.00017733762555020234,
      "logits/chosen": -0.49660688638687134,
      "logits/rejected": -0.5793140530586243,
      "logps/chosen": -7.395486354827881,
      "logps/rejected": -45.230892181396484,
      "loss": 0.4777700901031494,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3038936257362366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13568957149982452,
      "rewards/margins": 3.222832202911377,
      "rewards/rejected": -3.0871424674987793,
      "step": 231,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5177126917712692,
      "grad_norm": 1.362999439239502,
      "learning_rate": 0.0001771030680533697,
      "logits/chosen": -0.4924534261226654,
      "logits/rejected": -0.6207826733589172,
      "logps/chosen": -4.7806715965271,
      "logps/rejected": -48.792823791503906,
      "loss": 0.4094929099082947,
      "memory(GiB)": 46.81,
      "nll_loss": 0.2229592204093933,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2055450677871704,
      "rewards/margins": 3.34159779548645,
      "rewards/rejected": -3.1360526084899902,
      "step": 232,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5199442119944212,
      "grad_norm": 1.446892261505127,
      "learning_rate": 0.00017686745977655254,
      "logits/chosen": -0.5510218739509583,
      "logits/rejected": -0.5702377557754517,
      "logps/chosen": -7.457989692687988,
      "logps/rejected": -40.870391845703125,
      "loss": 0.6166619658470154,
      "memory(GiB)": 46.81,
      "nll_loss": 0.37686091661453247,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.019258877262473106,
      "rewards/margins": 2.9649226665496826,
      "rewards/rejected": -2.9456636905670166,
      "step": 233,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5221757322175732,
      "grad_norm": 2.5673022270202637,
      "learning_rate": 0.00017663080393067967,
      "logits/chosen": -0.4004722833633423,
      "logits/rejected": -0.6144905090332031,
      "logps/chosen": -8.209660530090332,
      "logps/rejected": -64.06336975097656,
      "loss": 0.4909789562225342,
      "memory(GiB)": 46.81,
      "nll_loss": 0.31386396288871765,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20118242502212524,
      "rewards/margins": 4.270333290100098,
      "rewards/rejected": -4.069150447845459,
      "step": 234,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5244072524407253,
      "grad_norm": 4.679993152618408,
      "learning_rate": 0.00017639310374095642,
      "logits/chosen": -0.47914889454841614,
      "logits/rejected": -0.6579112410545349,
      "logps/chosen": -8.027458190917969,
      "logps/rejected": -49.39167022705078,
      "loss": 0.6046391129493713,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4407365620136261,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19219349324703217,
      "rewards/margins": 3.430323362350464,
      "rewards/rejected": -3.2381296157836914,
      "step": 235,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5266387726638773,
      "grad_norm": 2.3275198936462402,
      "learning_rate": 0.00017615436244682067,
      "logits/chosen": -0.48543357849121094,
      "logits/rejected": -0.6728047132492065,
      "logps/chosen": -5.8822832107543945,
      "logps/rejected": -53.12212371826172,
      "loss": 0.7272516489028931,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5779345035552979,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11761157214641571,
      "rewards/margins": 3.723693370819092,
      "rewards/rejected": -3.606081962585449,
      "step": 236,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5288702928870292,
      "grad_norm": 1.9778822660446167,
      "learning_rate": 0.0001759145833018988,
      "logits/chosen": -0.5121138095855713,
      "logits/rejected": -0.581526517868042,
      "logps/chosen": -11.055684089660645,
      "logps/rejected": -48.428466796875,
      "loss": 0.6768224835395813,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4439318776130676,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07668354362249374,
      "rewards/margins": 3.0230700969696045,
      "rewards/rejected": -2.9463863372802734,
      "step": 237,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5311018131101813,
      "grad_norm": 2.473088264465332,
      "learning_rate": 0.00017567376957396125,
      "logits/chosen": -0.4359075427055359,
      "logits/rejected": -0.6152986288070679,
      "logps/chosen": -8.427140235900879,
      "logps/rejected": -43.27656555175781,
      "loss": 0.8277904391288757,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5778707265853882,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2914345860481262,
      "rewards/margins": 2.8861725330352783,
      "rewards/rejected": -2.594738006591797,
      "step": 238,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 4.2344560623168945,
      "learning_rate": 0.00017543192454487795,
      "logits/chosen": -0.5183901786804199,
      "logits/rejected": -0.648127555847168,
      "logps/chosen": -7.237810134887695,
      "logps/rejected": -46.60614776611328,
      "loss": 0.5062097311019897,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3680287003517151,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09910139441490173,
      "rewards/margins": 3.3295600414276123,
      "rewards/rejected": -3.2304584980010986,
      "step": 239,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5355648535564853,
      "grad_norm": 12.738329887390137,
      "learning_rate": 0.0001751890515105738,
      "logits/chosen": -0.5236122608184814,
      "logits/rejected": -0.6659674644470215,
      "logps/chosen": -6.250277042388916,
      "logps/rejected": -48.39547348022461,
      "loss": 0.8066755533218384,
      "memory(GiB)": 46.81,
      "nll_loss": 0.5374718904495239,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.011939210817217827,
      "rewards/margins": 3.1737723350524902,
      "rewards/rejected": -3.1618332862854004,
      "step": 240,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5377963737796374,
      "grad_norm": 1.7081589698791504,
      "learning_rate": 0.00017494515378098347,
      "logits/chosen": -0.49652621150016785,
      "logits/rejected": -0.6365297436714172,
      "logps/chosen": -4.525838375091553,
      "logps/rejected": -44.274539947509766,
      "loss": 0.4856886863708496,
      "memory(GiB)": 46.81,
      "nll_loss": 0.26593509316444397,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10261054337024689,
      "rewards/margins": 3.1593854427337646,
      "rewards/rejected": -3.056774616241455,
      "step": 241,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5400278940027894,
      "grad_norm": 1.5723114013671875,
      "learning_rate": 0.00017470023468000654,
      "logits/chosen": -0.5223122835159302,
      "logits/rejected": -0.6690611839294434,
      "logps/chosen": -3.2274744510650635,
      "logps/rejected": -53.48045349121094,
      "loss": 0.3347703516483307,
      "memory(GiB)": 46.81,
      "nll_loss": 0.19753773510456085,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1409139484167099,
      "rewards/margins": 3.965757369995117,
      "rewards/rejected": -3.824843406677246,
      "step": 242,
      "train_speed(iter/s)": 0.010691
    },
    {
      "epoch": 0.5422594142259414,
      "grad_norm": 1.1658532619476318,
      "learning_rate": 0.00017445429754546209,
      "logits/chosen": -0.4327467083930969,
      "logits/rejected": -0.5850556492805481,
      "logps/chosen": -6.455155372619629,
      "logps/rejected": -53.67913055419922,
      "loss": 0.45557722449302673,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3267042934894562,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24175924062728882,
      "rewards/margins": 3.549290418624878,
      "rewards/rejected": -3.3075313568115234,
      "step": 243,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5444909344490935,
      "grad_norm": 1.9491972923278809,
      "learning_rate": 0.00017420734572904309,
      "logits/chosen": -0.4307960569858551,
      "logits/rejected": -0.6243136525154114,
      "logps/chosen": -5.606278419494629,
      "logps/rejected": -51.735652923583984,
      "loss": 0.5139089226722717,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3616556227207184,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1640806794166565,
      "rewards/margins": 3.619340419769287,
      "rewards/rejected": -3.4552597999572754,
      "step": 244,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5467224546722455,
      "grad_norm": 2.2012929916381836,
      "learning_rate": 0.00017395938259627102,
      "logits/chosen": -0.343536376953125,
      "logits/rejected": -0.569451630115509,
      "logps/chosen": -5.0532708168029785,
      "logps/rejected": -62.23390197753906,
      "loss": 0.3923553228378296,
      "memory(GiB)": 46.81,
      "nll_loss": 0.29478776454925537,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23885516822338104,
      "rewards/margins": 4.132390975952148,
      "rewards/rejected": -3.893535852432251,
      "step": 245,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5489539748953974,
      "grad_norm": 3.378282308578491,
      "learning_rate": 0.00017371041152644976,
      "logits/chosen": -0.4609750807285309,
      "logits/rejected": -0.6062232851982117,
      "logps/chosen": -6.916518211364746,
      "logps/rejected": -52.16080856323242,
      "loss": 0.6823198199272156,
      "memory(GiB)": 46.81,
      "nll_loss": 0.45661455392837524,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.3001694083213806,
      "rewards/margins": 3.915588855743408,
      "rewards/rejected": -3.6154191493988037,
      "step": 246,
      "train_speed(iter/s)": 0.01069
    },
    {
      "epoch": 0.5511854951185495,
      "grad_norm": 2.7868404388427734,
      "learning_rate": 0.00017346043591261957,
      "logits/chosen": -0.4002017676830292,
      "logits/rejected": -0.6774240732192993,
      "logps/chosen": -7.5064496994018555,
      "logps/rejected": -64.324951171875,
      "loss": 0.5472509860992432,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4029460847377777,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3002982437610626,
      "rewards/margins": 4.144438743591309,
      "rewards/rejected": -3.8441405296325684,
      "step": 247,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5534170153417015,
      "grad_norm": 2.1542208194732666,
      "learning_rate": 0.0001732094591615109,
      "logits/chosen": -0.34761539101600647,
      "logits/rejected": -0.6237728595733643,
      "logps/chosen": -8.17116641998291,
      "logps/rejected": -72.38832092285156,
      "loss": 0.5291301012039185,
      "memory(GiB)": 46.81,
      "nll_loss": 0.3634389340877533,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2887139618396759,
      "rewards/margins": 4.893632888793945,
      "rewards/rejected": -4.604918956756592,
      "step": 248,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5556485355648536,
      "grad_norm": 5.7039337158203125,
      "learning_rate": 0.00017295748469349804,
      "logits/chosen": -0.44048506021499634,
      "logits/rejected": -0.7222636342048645,
      "logps/chosen": -7.497531890869141,
      "logps/rejected": -82.19099426269531,
      "loss": 0.7382796406745911,
      "memory(GiB)": 46.81,
      "nll_loss": 0.6829304695129395,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09070631116628647,
      "rewards/margins": 6.177928924560547,
      "rewards/rejected": -6.087222099304199,
      "step": 249,
      "train_speed(iter/s)": 0.010689
    },
    {
      "epoch": 0.5578800557880056,
      "grad_norm": 9.315430641174316,
      "learning_rate": 0.00017270451594255233,
      "logits/chosen": -0.5497281551361084,
      "logits/rejected": -0.6751365065574646,
      "logps/chosen": -8.304182052612305,
      "logps/rejected": -60.03815460205078,
      "loss": 0.5764181017875671,
      "memory(GiB)": 46.81,
      "nll_loss": 0.4564261734485626,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.06676550954580307,
      "rewards/margins": 4.525730133056641,
      "rewards/rejected": -4.592494964599609,
      "step": 250,
      "train_speed(iter/s)": 0.01069
    }
  ],
  "logging_steps": 1,
  "max_steps": 896,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.379183154429952e+17,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}