{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.602182913059842,
  "eval_steps": 300,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0060218291305984195,
      "grad_norm": 7.15871000289917,
      "learning_rate": 1.1764705882352942e-05,
      "logits/chosen": -0.32459306716918945,
      "logits/rejected": -0.5000401735305786,
      "logps/chosen": -3.3705074787139893,
      "logps/rejected": -19.54801368713379,
      "loss": 1.1142030954360962,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4210559129714966,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.005428
    },
    {
      "epoch": 0.012043658261196839,
      "grad_norm": 6.311911106109619,
      "learning_rate": 2.3529411764705884e-05,
      "logits/chosen": -0.41802966594696045,
      "logits/rejected": -0.5096024870872498,
      "logps/chosen": -8.828028678894043,
      "logps/rejected": -13.557395935058594,
      "loss": 1.4290869235992432,
      "memory(GiB)": 46.13,
      "nll_loss": 0.7359397411346436,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.005545
    },
    {
      "epoch": 0.018065487391795258,
      "grad_norm": 6.693342685699463,
      "learning_rate": 3.529411764705883e-05,
      "logits/chosen": -0.3694347143173218,
      "logits/rejected": -0.4798021912574768,
      "logps/chosen": -6.542965412139893,
      "logps/rejected": -17.62507438659668,
      "loss": 1.2220425605773926,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5347560048103333,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.007337508723139763,
      "rewards/margins": 0.011953208595514297,
      "rewards/rejected": -0.004615699406713247,
      "step": 3,
      "train_speed(iter/s)": 0.005572
    },
    {
      "epoch": 0.024087316522393678,
      "grad_norm": 4.8458251953125,
      "learning_rate": 4.705882352941177e-05,
      "logits/chosen": -0.36237555742263794,
      "logits/rejected": -0.5201014280319214,
      "logps/chosen": -5.53680944442749,
      "logps/rejected": -16.475929260253906,
      "loss": 1.2521781921386719,
      "memory(GiB)": 46.13,
      "nll_loss": 0.579364538192749,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.01270484272390604,
      "rewards/margins": 0.04416074603796005,
      "rewards/rejected": -0.03145590424537659,
      "step": 4,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.030109145652992095,
      "grad_norm": 3.684136152267456,
      "learning_rate": 5.882352941176471e-05,
      "logits/chosen": -0.3457816541194916,
      "logits/rejected": -0.48865967988967896,
      "logps/chosen": -4.437034606933594,
      "logps/rejected": -20.260162353515625,
      "loss": 0.9991621375083923,
      "memory(GiB)": 46.13,
      "nll_loss": 0.38671040534973145,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.002155877649784088,
      "rewards/margins": 0.2208954095840454,
      "rewards/rejected": -0.21873953938484192,
      "step": 5,
      "train_speed(iter/s)": 0.005606
    },
    {
      "epoch": 0.036130974783590515,
      "grad_norm": 6.8094706535339355,
      "learning_rate": 7.058823529411765e-05,
      "logits/chosen": -0.37496131658554077,
      "logits/rejected": -0.499918133020401,
      "logps/chosen": -5.312520980834961,
      "logps/rejected": -18.596221923828125,
      "loss": 1.1320774555206299,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5494582056999207,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.11241161823272705,
      "rewards/margins": 0.37036794424057007,
      "rewards/rejected": -0.4827795624732971,
      "step": 6,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.042152803914188935,
      "grad_norm": 6.563366889953613,
      "learning_rate": 8.23529411764706e-05,
      "logits/chosen": -0.3803170323371887,
      "logits/rejected": -0.43647128343582153,
      "logps/chosen": -12.155686378479004,
      "logps/rejected": -19.252357482910156,
      "loss": 1.275801420211792,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6294467449188232,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.2581723928451538,
      "rewards/margins": 0.26009732484817505,
      "rewards/rejected": -0.5182697176933289,
      "step": 7,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.048174633044787356,
      "grad_norm": 5.275461673736572,
      "learning_rate": 9.411764705882353e-05,
      "logits/chosen": -0.30787912011146545,
      "logits/rejected": -0.420376718044281,
      "logps/chosen": -7.271793842315674,
      "logps/rejected": -17.376087188720703,
      "loss": 1.2506061792373657,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6039291024208069,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.07998734712600708,
      "rewards/margins": 0.2555825710296631,
      "rewards/rejected": -0.3355698883533478,
      "step": 8,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.054196462175385776,
      "grad_norm": 3.0540287494659424,
      "learning_rate": 0.00010588235294117647,
      "logits/chosen": -0.30379173159599304,
      "logits/rejected": -0.41216912865638733,
      "logps/chosen": -6.820768356323242,
      "logps/rejected": -16.3853816986084,
      "loss": 1.1634117364883423,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5367689728736877,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.04989926144480705,
      "rewards/margins": 0.1963551640510559,
      "rewards/rejected": -0.14645591378211975,
      "step": 9,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.06021829130598419,
      "grad_norm": 2.3425395488739014,
      "learning_rate": 0.00011764705882352942,
      "logits/chosen": -0.3369804918766022,
      "logits/rejected": -0.4523884356021881,
      "logps/chosen": -7.222663402557373,
      "logps/rejected": -17.98470687866211,
      "loss": 1.0803574323654175,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4635474979877472,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0936385914683342,
      "rewards/margins": 0.19075465202331543,
      "rewards/rejected": -0.09711606800556183,
      "step": 10,
      "train_speed(iter/s)": 0.005616
    },
    {
      "epoch": 0.06624012043658262,
      "grad_norm": 2.2927637100219727,
      "learning_rate": 0.00012941176470588237,
      "logits/chosen": -0.37337860465049744,
      "logits/rejected": -0.4716382920742035,
      "logps/chosen": -8.593006134033203,
      "logps/rejected": -17.979745864868164,
      "loss": 1.1164621114730835,
      "memory(GiB)": 46.13,
      "nll_loss": 0.49796244502067566,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.06070924177765846,
      "rewards/margins": 0.19886666536331177,
      "rewards/rejected": -0.1381574273109436,
      "step": 11,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.07226194956718103,
      "grad_norm": 2.380297899246216,
      "learning_rate": 0.0001411764705882353,
      "logits/chosen": -0.4056618809700012,
      "logits/rejected": -0.47582757472991943,
      "logps/chosen": -7.429853916168213,
      "logps/rejected": -16.317821502685547,
      "loss": 1.167059302330017,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5555555820465088,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.0010902846697717905,
      "rewards/margins": 0.21717122197151184,
      "rewards/rejected": -0.2182615101337433,
      "step": 12,
      "train_speed(iter/s)": 0.005622
    },
    {
      "epoch": 0.07828377869777944,
      "grad_norm": 1.7392877340316772,
      "learning_rate": 0.00015294117647058822,
      "logits/chosen": -0.4110736846923828,
      "logits/rejected": -0.4770917296409607,
      "logps/chosen": -8.422821044921875,
      "logps/rejected": -16.356956481933594,
      "loss": 1.1002569198608398,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5357962250709534,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.12992499768733978,
      "rewards/margins": 0.3372637927532196,
      "rewards/rejected": -0.20733879506587982,
      "step": 13,
      "train_speed(iter/s)": 0.005625
    },
    {
      "epoch": 0.08430560782837787,
      "grad_norm": 5.562183856964111,
      "learning_rate": 0.0001647058823529412,
      "logits/chosen": -0.32676243782043457,
      "logits/rejected": -0.4272843599319458,
      "logps/chosen": -8.393424034118652,
      "logps/rejected": -19.974212646484375,
      "loss": 1.156798005104065,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6415989995002747,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.03989575803279877,
      "rewards/margins": 0.5587274432182312,
      "rewards/rejected": -0.5986231565475464,
      "step": 14,
      "train_speed(iter/s)": 0.005627
    },
    {
      "epoch": 0.09032743695897628,
      "grad_norm": 3.5578579902648926,
      "learning_rate": 0.00017647058823529413,
      "logits/chosen": -0.36884137988090515,
      "logits/rejected": -0.47317999601364136,
      "logps/chosen": -6.998829364776611,
      "logps/rejected": -18.39483642578125,
      "loss": 1.265438199043274,
      "memory(GiB)": 46.13,
      "nll_loss": 0.7606177926063538,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.002129599452018738,
      "rewards/margins": 0.5497386455535889,
      "rewards/rejected": -0.5476090908050537,
      "step": 15,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.09634926608957471,
      "grad_norm": 3.6414742469787598,
      "learning_rate": 0.00018823529411764707,
      "logits/chosen": -0.3789006471633911,
      "logits/rejected": -0.4743453860282898,
      "logps/chosen": -6.895637512207031,
      "logps/rejected": -20.256675720214844,
      "loss": 1.064023494720459,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5308047533035278,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.055957891047000885,
      "rewards/margins": 0.40152865648269653,
      "rewards/rejected": -0.34557074308395386,
      "step": 16,
      "train_speed(iter/s)": 0.005632
    },
    {
      "epoch": 0.10237109522017313,
      "grad_norm": 3.263599157333374,
      "learning_rate": 0.0002,
      "logits/chosen": -0.28460752964019775,
      "logits/rejected": -0.42314791679382324,
      "logps/chosen": -5.295862197875977,
      "logps/rejected": -21.666519165039062,
      "loss": 0.9750788807868958,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5031211376190186,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08322687447071075,
      "rewards/margins": 0.5906113386154175,
      "rewards/rejected": -0.5073844194412231,
      "step": 17,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.10839292435077155,
      "grad_norm": 2.0240275859832764,
      "learning_rate": 0.00019999502669559432,
      "logits/chosen": -0.3283552825450897,
      "logits/rejected": -0.40596985816955566,
      "logps/chosen": -6.424624919891357,
      "logps/rejected": -17.71906852722168,
      "loss": 0.9781989455223083,
      "memory(GiB)": 46.13,
      "nll_loss": 0.47217291593551636,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.0207663644105196,
      "rewards/margins": 0.5476341247558594,
      "rewards/rejected": -0.568400502204895,
      "step": 18,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.11441475348136997,
      "grad_norm": 1.682889699935913,
      "learning_rate": 0.00019998010727705236,
      "logits/chosen": -0.2821420729160309,
      "logits/rejected": -0.38918256759643555,
      "logps/chosen": -5.882609844207764,
      "logps/rejected": -20.25113296508789,
      "loss": 0.9472936391830444,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4797293543815613,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.06502628326416016,
      "rewards/margins": 0.6772950291633606,
      "rewards/rejected": -0.6122686862945557,
      "step": 19,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.12043658261196838,
      "grad_norm": 2.9386143684387207,
      "learning_rate": 0.00019995524322835034,
      "logits/chosen": -0.2649421691894531,
      "logits/rejected": -0.34010952711105347,
      "logps/chosen": -10.95669174194336,
      "logps/rejected": -24.044157028198242,
      "loss": 1.1134816408157349,
      "memory(GiB)": 46.13,
      "nll_loss": 0.546178936958313,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.18016880750656128,
      "rewards/margins": 0.5494988560676575,
      "rewards/rejected": -0.729667603969574,
      "step": 20,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.1264584117425668,
      "grad_norm": 1.6700323820114136,
      "learning_rate": 0.00019992043702261793,
      "logits/chosen": -0.2561277151107788,
      "logits/rejected": -0.3126121163368225,
      "logps/chosen": -6.485073566436768,
      "logps/rejected": -19.772449493408203,
      "loss": 0.9205772876739502,
      "memory(GiB)": 46.13,
      "nll_loss": 0.42214635014533997,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.0028960183262825012,
      "rewards/margins": 0.6638913750648499,
      "rewards/rejected": -0.6667873859405518,
      "step": 21,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.13248024087316523,
      "grad_norm": 2.0114383697509766,
      "learning_rate": 0.00019987569212189224,
      "logits/chosen": -0.2601704001426697,
      "logits/rejected": -0.3789623975753784,
      "logps/chosen": -6.645152568817139,
      "logps/rejected": -21.876020431518555,
      "loss": 1.1017377376556396,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6103405356407166,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.04465393349528313,
      "rewards/margins": 0.6134660840034485,
      "rewards/rejected": -0.5688121318817139,
      "step": 22,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.13850207000376363,
      "grad_norm": 3.031888961791992,
      "learning_rate": 0.0001998210129767735,
      "logits/chosen": -0.26405903697013855,
      "logits/rejected": -0.3669392466545105,
      "logps/chosen": -6.921745300292969,
      "logps/rejected": -21.70663070678711,
      "loss": 0.9795225858688354,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4959058463573456,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.10062012821435928,
      "rewards/margins": 0.6365569829940796,
      "rewards/rejected": -0.5359368324279785,
      "step": 23,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.14452389913436206,
      "grad_norm": 2.15742564201355,
      "learning_rate": 0.00019975640502598244,
      "logits/chosen": -0.2354922890663147,
      "logits/rejected": -0.36152032017707825,
      "logps/chosen": -4.718395709991455,
      "logps/rejected": -24.57033348083496,
      "loss": 0.7706338763237,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3560357987880707,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.09812817722558975,
      "rewards/margins": 0.9238024950027466,
      "rewards/rejected": -0.825674295425415,
      "step": 24,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.1505457282649605,
      "grad_norm": 3.1512320041656494,
      "learning_rate": 0.0001996818746958191,
      "logits/chosen": -0.21456243097782135,
      "logits/rejected": -0.34130796790122986,
      "logps/chosen": -6.996722221374512,
      "logps/rejected": -30.622554779052734,
      "loss": 0.8399394750595093,
      "memory(GiB)": 46.13,
      "nll_loss": 0.49700063467025757,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.024456650018692017,
      "rewards/margins": 1.4392318725585938,
      "rewards/rejected": -1.4636887311935425,
      "step": 25,
      "train_speed(iter/s)": 0.005628
    },
    {
      "epoch": 0.1565675573955589,
      "grad_norm": 2.9886889457702637,
      "learning_rate": 0.00019959742939952392,
      "logits/chosen": -0.2622988224029541,
      "logits/rejected": -0.3251183331012726,
      "logps/chosen": -12.495084762573242,
      "logps/rejected": -25.176090240478516,
      "loss": 1.1550263166427612,
      "memory(GiB)": 46.13,
      "nll_loss": 0.7094165086746216,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.19754531979560852,
      "rewards/margins": 1.1228585243225098,
      "rewards/rejected": -1.3204039335250854,
      "step": 26,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.16258938652615731,
      "grad_norm": 2.7348685264587402,
      "learning_rate": 0.00019950307753654017,
      "logits/chosen": -0.23797279596328735,
      "logits/rejected": -0.3161388337612152,
      "logps/chosen": -8.345640182495117,
      "logps/rejected": -24.480266571044922,
      "loss": 1.0113354921340942,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5643945336341858,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.05142675340175629,
      "rewards/margins": 1.0910050868988037,
      "rewards/rejected": -1.0395784378051758,
      "step": 27,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.16861121565675574,
      "grad_norm": 3.4209959506988525,
      "learning_rate": 0.00019939882849167852,
      "logits/chosen": -0.2834317684173584,
      "logits/rejected": -0.4013361930847168,
      "logps/chosen": -6.27006196975708,
      "logps/rejected": -23.9411678314209,
      "loss": 0.9620530605316162,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5452341437339783,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08509417623281479,
      "rewards/margins": 1.0887510776519775,
      "rewards/rejected": -1.0036569833755493,
      "step": 28,
      "train_speed(iter/s)": 0.005632
    },
    {
      "epoch": 0.17463304478735417,
      "grad_norm": 3.021099805831909,
      "learning_rate": 0.00019928469263418374,
      "logits/chosen": -0.29641368985176086,
      "logits/rejected": -0.3944772779941559,
      "logps/chosen": -6.065124988555908,
      "logps/rejected": -19.367380142211914,
      "loss": 0.8545750379562378,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3696524500846863,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.008128602989017963,
      "rewards/margins": 0.7214774489402771,
      "rewards/rejected": -0.729606032371521,
      "step": 29,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.18065487391795257,
      "grad_norm": 2.237170457839966,
      "learning_rate": 0.00019916068131670302,
      "logits/chosen": -0.2665432393550873,
      "logits/rejected": -0.41999131441116333,
      "logps/chosen": -3.6829960346221924,
      "logps/rejected": -24.998981475830078,
      "loss": 0.7719818353652954,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3786780834197998,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.15382343530654907,
      "rewards/margins": 1.0922845602035522,
      "rewards/rejected": -0.9384610652923584,
      "step": 30,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.186676703048551,
      "grad_norm": 2.299628496170044,
      "learning_rate": 0.00019902680687415705,
      "logits/chosen": -0.33755987882614136,
      "logits/rejected": -0.37323832511901855,
      "logps/chosen": -11.234481811523438,
      "logps/rejected": -24.92943000793457,
      "loss": 1.0785183906555176,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6123806834220886,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.03546600416302681,
      "rewards/margins": 0.9260610342025757,
      "rewards/rejected": -0.890595018863678,
      "step": 31,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.19269853217914942,
      "grad_norm": 2.0755157470703125,
      "learning_rate": 0.00019888308262251285,
      "logits/chosen": -0.2504952847957611,
      "logits/rejected": -0.3852018713951111,
      "logps/chosen": -5.893071174621582,
      "logps/rejected": -33.010154724121094,
      "loss": 0.7333768010139465,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4039708375930786,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.07270973920822144,
      "rewards/margins": 1.7266591787338257,
      "rewards/rejected": -1.7993686199188232,
      "step": 32,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.19872036130974782,
      "grad_norm": 3.686795711517334,
      "learning_rate": 0.00019872952285745959,
      "logits/chosen": -0.30149099230766296,
      "logits/rejected": -0.4157900810241699,
      "logps/chosen": -7.993706703186035,
      "logps/rejected": -27.117816925048828,
      "loss": 1.0753562450408936,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6016828417778015,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.10546419769525528,
      "rewards/margins": 1.0370161533355713,
      "rewards/rejected": -1.1424803733825684,
      "step": 33,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.20474219044034625,
      "grad_norm": 3.085048198699951,
      "learning_rate": 0.0001985661428529863,
      "logits/chosen": -0.31711238622665405,
      "logits/rejected": -0.41008925437927246,
      "logps/chosen": -6.8641557693481445,
      "logps/rejected": -27.393281936645508,
      "loss": 1.054256558418274,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6109166741371155,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.06303416192531586,
      "rewards/margins": 1.1662019491195679,
      "rewards/rejected": -1.229236125946045,
      "step": 34,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.21076401957094468,
      "grad_norm": 1.7560195922851562,
      "learning_rate": 0.00019839295885986296,
      "logits/chosen": -0.337604820728302,
      "logits/rejected": -0.39799487590789795,
      "logps/chosen": -6.864170551300049,
      "logps/rejected": -20.806344985961914,
      "loss": 0.9619123339653015,
      "memory(GiB)": 46.13,
      "nll_loss": 0.48513123393058777,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.10183387249708176,
      "rewards/margins": 0.7188172340393066,
      "rewards/rejected": -0.6169832944869995,
      "step": 35,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.2167858487015431,
      "grad_norm": 2.6364471912384033,
      "learning_rate": 0.0001982099881040239,
      "logits/chosen": -0.36966580152511597,
      "logits/rejected": -0.43631935119628906,
      "logps/chosen": -6.647594451904297,
      "logps/rejected": -21.291913986206055,
      "loss": 0.9380239844322205,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4613775312900543,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.06358329951763153,
      "rewards/margins": 0.7893975973129272,
      "rewards/rejected": -0.7258143424987793,
      "step": 36,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.2228076778321415,
      "grad_norm": 2.354177474975586,
      "learning_rate": 0.00019801724878485438,
      "logits/chosen": -0.36403897404670715,
      "logits/rejected": -0.45796939730644226,
      "logps/chosen": -5.62083625793457,
      "logps/rejected": -23.648834228515625,
      "loss": 0.9243438243865967,
      "memory(GiB)": 46.13,
      "nll_loss": 0.46970799565315247,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.06422886252403259,
      "rewards/margins": 0.8610265851020813,
      "rewards/rejected": -0.7967977523803711,
      "step": 37,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.22882950696273993,
      "grad_norm": 2.2988836765289307,
      "learning_rate": 0.00019781476007338058,
      "logits/chosen": -0.3704385459423065,
      "logits/rejected": -0.47973009943962097,
      "logps/chosen": -4.231719017028809,
      "logps/rejected": -20.56240463256836,
      "loss": 0.916353702545166,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4209356904029846,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.06186368316411972,
      "rewards/margins": 0.7635293006896973,
      "rewards/rejected": -0.7016656398773193,
      "step": 38,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.23485133609333836,
      "grad_norm": 2.009106397628784,
      "learning_rate": 0.00019760254211036244,
      "logits/chosen": -0.3372359573841095,
      "logits/rejected": -0.453107088804245,
      "logps/chosen": -5.785426616668701,
      "logps/rejected": -23.19124412536621,
      "loss": 0.9124959707260132,
      "memory(GiB)": 46.13,
      "nll_loss": 0.44838690757751465,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.04832564294338226,
      "rewards/margins": 0.937610387802124,
      "rewards/rejected": -0.8892846703529358,
      "step": 39,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.24087316522393676,
      "grad_norm": 2.5412213802337646,
      "learning_rate": 0.00019738061600429064,
      "logits/chosen": -0.39427676796913147,
      "logits/rejected": -0.48203372955322266,
      "logps/chosen": -5.785412311553955,
      "logps/rejected": -19.031665802001953,
      "loss": 0.9683349132537842,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5110080242156982,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.022132933139801025,
      "rewards/margins": 0.8074027299880981,
      "rewards/rejected": -0.7852697372436523,
      "step": 40,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.24689499435453519,
      "grad_norm": 2.0049972534179688,
      "learning_rate": 0.00019714900382928675,
      "logits/chosen": -0.3044354021549225,
      "logits/rejected": -0.41962650418281555,
      "logps/chosen": -4.612288475036621,
      "logps/rejected": -24.228839874267578,
      "loss": 0.8179229497909546,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4278942048549652,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07943032681941986,
      "rewards/margins": 1.1561895608901978,
      "rewards/rejected": -1.0767593383789062,
      "step": 41,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.2529168234851336,
      "grad_norm": 2.0874667167663574,
      "learning_rate": 0.0001969077286229078,
      "logits/chosen": -0.2736699879169464,
      "logits/rejected": -0.4168338179588318,
      "logps/chosen": -7.539179801940918,
      "logps/rejected": -30.339994430541992,
      "loss": 0.9962757229804993,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6084927320480347,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.046928200870752335,
      "rewards/margins": 1.4784610271453857,
      "rewards/rejected": -1.431532859802246,
      "step": 42,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.258938652615732,
      "grad_norm": 3.2748661041259766,
      "learning_rate": 0.00019665681438385473,
      "logits/chosen": -0.26249000430107117,
      "logits/rejected": -0.3955192565917969,
      "logps/chosen": -6.726841449737549,
      "logps/rejected": -26.447267532348633,
      "loss": 0.9923527836799622,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5636712312698364,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.02616805024445057,
      "rewards/margins": 1.2797741889953613,
      "rewards/rejected": -1.3059422969818115,
      "step": 43,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.26496048174633047,
      "grad_norm": 4.499722957611084,
      "learning_rate": 0.00019639628606958533,
      "logits/chosen": -0.3380294442176819,
      "logits/rejected": -0.4303723871707916,
      "logps/chosen": -7.925315856933594,
      "logps/rejected": -23.916906356811523,
      "loss": 1.136991024017334,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6508135795593262,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.1142488494515419,
      "rewards/margins": 1.0733367204666138,
      "rewards/rejected": -1.1875855922698975,
      "step": 44,
      "train_speed(iter/s)": 0.005638
    },
    {
      "epoch": 0.27098231087692887,
      "grad_norm": 3.695734739303589,
      "learning_rate": 0.0001961261695938319,
      "logits/chosen": -0.2465212196111679,
      "logits/rejected": -0.39715391397476196,
      "logps/chosen": -7.8324666023254395,
      "logps/rejected": -24.97869110107422,
      "loss": 0.9623777270317078,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5307109951972961,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.08475106954574585,
      "rewards/margins": 0.9390297532081604,
      "rewards/rejected": -0.8542786836624146,
      "step": 45,
      "train_speed(iter/s)": 0.005638
    },
    {
      "epoch": 0.27700414000752727,
      "grad_norm": 3.1841602325439453,
      "learning_rate": 0.00019584649182402357,
      "logits/chosen": -0.22775670886039734,
      "logits/rejected": -0.3454330265522003,
      "logps/chosen": -6.5436859130859375,
      "logps/rejected": -24.402027130126953,
      "loss": 1.0284608602523804,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5513083338737488,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.05229588598012924,
      "rewards/margins": 0.9507507085800171,
      "rewards/rejected": -0.8984547853469849,
      "step": 46,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.2830259691381257,
      "grad_norm": 2.0325071811676025,
      "learning_rate": 0.0001955572805786141,
      "logits/chosen": -0.19194024801254272,
      "logits/rejected": -0.32474127411842346,
      "logps/chosen": -4.646670341491699,
      "logps/rejected": -27.415727615356445,
      "loss": 0.7930896878242493,
      "memory(GiB)": 46.13,
      "nll_loss": 0.38733866810798645,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.11078543961048126,
      "rewards/margins": 1.1206859350204468,
      "rewards/rejected": -1.009900450706482,
      "step": 47,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.2890477982687241,
      "grad_norm": 2.07770037651062,
      "learning_rate": 0.0001952585646243146,
      "logits/chosen": -0.25676336884498596,
      "logits/rejected": -0.3494555950164795,
      "logps/chosen": -7.388433933258057,
      "logps/rejected": -23.070579528808594,
      "loss": 0.8534475564956665,
      "memory(GiB)": 46.13,
      "nll_loss": 0.42261388897895813,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.006194199435412884,
      "rewards/margins": 0.9558709859848022,
      "rewards/rejected": -0.9496768712997437,
      "step": 48,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.2950696273993225,
      "grad_norm": 1.6862146854400635,
      "learning_rate": 0.00019495037367323262,
      "logits/chosen": -0.22853028774261475,
      "logits/rejected": -0.3647540211677551,
      "logps/chosen": -3.3972930908203125,
      "logps/rejected": -25.391393661499023,
      "loss": 0.7584111094474792,
      "memory(GiB)": 46.13,
      "nll_loss": 0.35165295004844666,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.12043334543704987,
      "rewards/margins": 1.1430784463882446,
      "rewards/rejected": -1.022645115852356,
      "step": 49,
      "train_speed(iter/s)": 0.005638
    },
    {
      "epoch": 0.301091456529921,
      "grad_norm": 3.027331829071045,
      "learning_rate": 0.00019463273837991643,
      "logits/chosen": -0.2110910564661026,
      "logits/rejected": -0.38482916355133057,
      "logps/chosen": -4.55904483795166,
      "logps/rejected": -31.877506256103516,
      "loss": 0.675049901008606,
      "memory(GiB)": 46.13,
      "nll_loss": 0.32799023389816284,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05019056051969528,
      "rewards/margins": 1.6790026426315308,
      "rewards/rejected": -1.6288120746612549,
      "step": 50,
      "train_speed(iter/s)": 0.005638
    },
    {
      "epoch": 0.3071132856605194,
      "grad_norm": 2.950259208679199,
      "learning_rate": 0.00019430569033830605,
      "logits/chosen": -0.19638891518115997,
      "logits/rejected": -0.3516000807285309,
      "logps/chosen": -7.3598952293396,
      "logps/rejected": -32.14203643798828,
      "loss": 0.9957353472709656,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6411410570144653,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.031813204288482666,
      "rewards/margins": 1.6832952499389648,
      "rewards/rejected": -1.7151083946228027,
      "step": 51,
      "train_speed(iter/s)": 0.005628
    },
    {
      "epoch": 0.3131351147911178,
      "grad_norm": 3.4817123413085938,
      "learning_rate": 0.00019396926207859084,
      "logits/chosen": -0.2925015687942505,
      "logits/rejected": -0.4453135132789612,
      "logps/chosen": -6.357159614562988,
      "logps/rejected": -34.39080047607422,
      "loss": 0.9899907112121582,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6184481382369995,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.06884388625621796,
      "rewards/margins": 1.8842136859893799,
      "rewards/rejected": -1.9530572891235352,
      "step": 52,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.31915694392171623,
      "grad_norm": 3.4239518642425537,
      "learning_rate": 0.00019362348706397373,
      "logits/chosen": -0.22639772295951843,
      "logits/rejected": -0.4221537709236145,
      "logps/chosen": -5.600902080535889,
      "logps/rejected": -36.40532684326172,
      "loss": 0.8088786602020264,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4771510660648346,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.0569700188934803,
      "rewards/margins": 1.8676366806030273,
      "rewards/rejected": -1.810666561126709,
      "step": 53,
      "train_speed(iter/s)": 0.005628
    },
    {
      "epoch": 0.32517877305231463,
      "grad_norm": 3.8685495853424072,
      "learning_rate": 0.00019326839968734279,
      "logits/chosen": -0.2569487690925598,
      "logits/rejected": -0.40343666076660156,
      "logps/chosen": -5.900696277618408,
      "logps/rejected": -36.28150177001953,
      "loss": 0.7994194030761719,
      "memory(GiB)": 46.13,
      "nll_loss": 0.43986907601356506,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.030100831761956215,
      "rewards/margins": 1.9989190101623535,
      "rewards/rejected": -1.968818187713623,
      "step": 54,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.3312006021829131,
      "grad_norm": 3.0355138778686523,
      "learning_rate": 0.00019290403526785025,
      "logits/chosen": -0.25347286462783813,
      "logits/rejected": -0.3757534623146057,
      "logps/chosen": -7.442959308624268,
      "logps/rejected": -32.45537567138672,
      "loss": 0.903397798538208,
      "memory(GiB)": 46.13,
      "nll_loss": 0.531738817691803,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.0414905920624733,
      "rewards/margins": 1.6564455032348633,
      "rewards/rejected": -1.614954948425293,
      "step": 55,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.3372224313135115,
      "grad_norm": 3.739659070968628,
      "learning_rate": 0.00019253043004739968,
      "logits/chosen": -0.20877386629581451,
      "logits/rejected": -0.37541496753692627,
      "logps/chosen": -7.142071723937988,
      "logps/rejected": -32.15766906738281,
      "loss": 0.9619933366775513,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4930208921432495,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.08227778971195221,
      "rewards/margins": 1.5348973274230957,
      "rewards/rejected": -1.6171751022338867,
      "step": 56,
      "train_speed(iter/s)": 0.005628
    },
    {
      "epoch": 0.3432442604441099,
      "grad_norm": 2.4167847633361816,
      "learning_rate": 0.00019214762118704076,
      "logits/chosen": -0.32553619146347046,
      "logits/rejected": -0.44049111008644104,
      "logps/chosen": -6.96627140045166,
      "logps/rejected": -24.777843475341797,
      "loss": 0.8523041605949402,
      "memory(GiB)": 46.13,
      "nll_loss": 0.44423431158065796,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.08760333806276321,
      "rewards/margins": 1.0987879037857056,
      "rewards/rejected": -1.011184573173523,
      "step": 57,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.34926608957470834,
      "grad_norm": 1.849902629852295,
      "learning_rate": 0.00019175564676327339,
      "logits/chosen": -0.29927390813827515,
      "logits/rejected": -0.4095182418823242,
      "logps/chosen": -6.824729919433594,
      "logps/rejected": -31.636598587036133,
      "loss": 0.827016294002533,
      "memory(GiB)": 46.13,
      "nll_loss": 0.432888925075531,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.10196143388748169,
      "rewards/margins": 1.6724345684051514,
      "rewards/rejected": -1.5704729557037354,
      "step": 58,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.35528791870530674,
      "grad_norm": 2.3601560592651367,
      "learning_rate": 0.0001913545457642601,
      "logits/chosen": -0.31936806440353394,
      "logits/rejected": -0.42600923776626587,
      "logps/chosen": -6.396060943603516,
      "logps/rejected": -26.946306228637695,
      "loss": 0.9836419224739075,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5654138326644897,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.013314278796315193,
      "rewards/margins": 1.2707021236419678,
      "rewards/rejected": -1.2573878765106201,
      "step": 59,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.36130974783590514,
      "grad_norm": 2.7022287845611572,
      "learning_rate": 0.00019094435808594823,
      "logits/chosen": -0.3565162420272827,
      "logits/rejected": -0.48129239678382874,
      "logps/chosen": -4.523666858673096,
      "logps/rejected": -30.854751586914062,
      "loss": 0.7463247776031494,
      "memory(GiB)": 46.13,
      "nll_loss": 0.41281038522720337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06385684013366699,
      "rewards/margins": 1.8442981243133545,
      "rewards/rejected": -1.7804412841796875,
      "step": 60,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.3673315769665036,
      "grad_norm": 2.549546957015991,
      "learning_rate": 0.0001905251245281015,
      "logits/chosen": -0.2870449125766754,
      "logits/rejected": -0.4699952006340027,
      "logps/chosen": -3.242504596710205,
      "logps/rejected": -35.58238220214844,
      "loss": 0.6698994636535645,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3660232126712799,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.03870222344994545,
      "rewards/margins": 1.8160982131958008,
      "rewards/rejected": -1.7773959636688232,
      "step": 61,
      "train_speed(iter/s)": 0.005631
    },
    {
      "epoch": 0.373353406097102,
      "grad_norm": 3.344719409942627,
      "learning_rate": 0.0001900968867902419,
      "logits/chosen": -0.26915398240089417,
      "logits/rejected": -0.38059523701667786,
      "logps/chosen": -6.996611595153809,
      "logps/rejected": -36.200714111328125,
      "loss": 0.8631665110588074,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5069284439086914,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.0650690495967865,
      "rewards/margins": 1.7960646152496338,
      "rewards/rejected": -1.8611338138580322,
      "step": 62,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.3793752352277004,
      "grad_norm": 1.7888697385787964,
      "learning_rate": 0.0001896596874675021,
      "logits/chosen": -0.25557130575180054,
      "logits/rejected": -0.4095707833766937,
      "logps/chosen": -5.574047088623047,
      "logps/rejected": -31.464292526245117,
      "loss": 0.6977128386497498,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3594370484352112,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08366499841213226,
      "rewards/margins": 1.6534584760665894,
      "rewards/rejected": -1.569793462753296,
      "step": 63,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.38539706435829885,
      "grad_norm": 2.8035733699798584,
      "learning_rate": 0.00018921357004638835,
      "logits/chosen": -0.22736577689647675,
      "logits/rejected": -0.3719921410083771,
      "logps/chosen": -6.317353248596191,
      "logps/rejected": -30.95716094970703,
      "loss": 0.8575116991996765,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5229077339172363,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13013550639152527,
      "rewards/margins": 1.687712550163269,
      "rewards/rejected": -1.5575770139694214,
      "step": 64,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.39141889348889725,
      "grad_norm": 2.560202121734619,
      "learning_rate": 0.00018875857890045543,
      "logits/chosen": -0.24293088912963867,
      "logits/rejected": -0.35710442066192627,
      "logps/chosen": -7.503670692443848,
      "logps/rejected": -32.99870300292969,
      "loss": 0.8683996796607971,
      "memory(GiB)": 46.13,
      "nll_loss": 0.49205920100212097,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.03412435203790665,
      "rewards/margins": 1.4795939922332764,
      "rewards/rejected": -1.4454697370529175,
      "step": 65,
      "train_speed(iter/s)": 0.005629
    },
    {
      "epoch": 0.39744072261949565,
      "grad_norm": 1.993003487586975,
      "learning_rate": 0.00018829475928589271,
      "logits/chosen": -0.23541702330112457,
      "logits/rejected": -0.3272259533405304,
      "logps/chosen": -5.38151216506958,
      "logps/rejected": -29.121570587158203,
      "loss": 0.7320932149887085,
      "memory(GiB)": 46.13,
      "nll_loss": 0.36119604110717773,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.0803036242723465,
      "rewards/margins": 1.6109750270843506,
      "rewards/rejected": -1.53067147731781,
      "step": 66,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.4034625517500941,
      "grad_norm": 2.9964606761932373,
      "learning_rate": 0.00018782215733702286,
      "logits/chosen": -0.20383372902870178,
      "logits/rejected": -0.34373044967651367,
      "logps/chosen": -6.649240493774414,
      "logps/rejected": -31.494646072387695,
      "loss": 0.8884384036064148,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5308991074562073,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.10617589205503464,
      "rewards/margins": 1.5828511714935303,
      "rewards/rejected": -1.4766751527786255,
      "step": 67,
      "train_speed(iter/s)": 0.00563
    },
    {
      "epoch": 0.4094843808806925,
      "grad_norm": 2.774994134902954,
      "learning_rate": 0.00018734082006171299,
      "logits/chosen": -0.25100839138031006,
      "logits/rejected": -0.4020751118659973,
      "logps/chosen": -7.144831657409668,
      "logps/rejected": -29.9566707611084,
      "loss": 0.8242477178573608,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5139617919921875,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13794061541557312,
      "rewards/margins": 1.6364552974700928,
      "rewards/rejected": -1.4985146522521973,
      "step": 68,
      "train_speed(iter/s)": 0.005631
    },
    {
      "epoch": 0.41550621001129096,
      "grad_norm": 3.4358551502227783,
      "learning_rate": 0.0001868507953366989,
      "logits/chosen": -0.24417072534561157,
      "logits/rejected": -0.36953434348106384,
      "logps/chosen": -6.3338799476623535,
      "logps/rejected": -32.21156311035156,
      "loss": 0.7758980989456177,
      "memory(GiB)": 46.13,
      "nll_loss": 0.48221129179000854,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14049482345581055,
      "rewards/margins": 1.806667447090149,
      "rewards/rejected": -1.666172742843628,
      "step": 69,
      "train_speed(iter/s)": 0.005631
    },
    {
      "epoch": 0.42152803914188935,
      "grad_norm": 2.821110248565674,
      "learning_rate": 0.0001863521319028231,
      "logits/chosen": -0.2586533725261688,
      "logits/rejected": -0.36297786235809326,
      "logps/chosen": -7.11786413192749,
      "logps/rejected": -31.115772247314453,
      "loss": 0.8807559013366699,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5770330429077148,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.07292432337999344,
      "rewards/margins": 2.026425838470459,
      "rewards/rejected": -1.9535014629364014,
      "step": 70,
      "train_speed(iter/s)": 0.005632
    },
    {
      "epoch": 0.42754986827248775,
      "grad_norm": 2.6153159141540527,
      "learning_rate": 0.00018584487936018661,
      "logits/chosen": -0.305365651845932,
      "logits/rejected": -0.4076971709728241,
      "logps/chosen": -4.401376247406006,
      "logps/rejected": -25.76744842529297,
      "loss": 0.7574676275253296,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3759956955909729,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.015199379995465279,
      "rewards/margins": 1.5401445627212524,
      "rewards/rejected": -1.5553438663482666,
      "step": 71,
      "train_speed(iter/s)": 0.005632
    },
    {
      "epoch": 0.4335716974030862,
      "grad_norm": 4.708693027496338,
      "learning_rate": 0.00018532908816321558,
      "logits/chosen": -0.22257624566555023,
      "logits/rejected": -0.36776047945022583,
      "logps/chosen": -5.041079998016357,
      "logps/rejected": -31.822460174560547,
      "loss": 0.7604058980941772,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3996947705745697,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0631202757358551,
      "rewards/margins": 1.8200846910476685,
      "rewards/rejected": -1.7569644451141357,
      "step": 72,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.4395935265336846,
      "grad_norm": 4.16855001449585,
      "learning_rate": 0.0001848048096156426,
      "logits/chosen": -0.1833086460828781,
      "logits/rejected": -0.40944039821624756,
      "logps/chosen": -7.540615081787109,
      "logps/rejected": -41.107208251953125,
      "loss": 1.1337863206863403,
      "memory(GiB)": 46.13,
      "nll_loss": 0.7324954271316528,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.13253508508205414,
      "rewards/margins": 2.3366596698760986,
      "rewards/rejected": -2.4691946506500244,
      "step": 73,
      "train_speed(iter/s)": 0.005633
    },
    {
      "epoch": 0.445615355664283,
      "grad_norm": 2.806788444519043,
      "learning_rate": 0.0001842720958654039,
      "logits/chosen": -0.3181215524673462,
      "logits/rejected": -0.39911144971847534,
      "logps/chosen": -5.607314586639404,
      "logps/rejected": -28.428564071655273,
      "loss": 0.6960507035255432,
      "memory(GiB)": 46.13,
      "nll_loss": 0.40211787819862366,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15752148628234863,
      "rewards/margins": 1.9513351917266846,
      "rewards/rejected": -1.7938138246536255,
      "step": 74,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.45163718479488146,
      "grad_norm": 3.420902967453003,
      "learning_rate": 0.00018373099989945236,
      "logits/chosen": -0.22940213978290558,
      "logits/rejected": -0.4191284477710724,
      "logps/chosen": -6.713252067565918,
      "logps/rejected": -37.6962890625,
      "loss": 0.920997679233551,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5668741464614868,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": -0.0326458215713501,
      "rewards/margins": 2.1453044414520264,
      "rewards/rejected": -2.177950382232666,
      "step": 75,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.45765901392547986,
      "grad_norm": 3.6164190769195557,
      "learning_rate": 0.0001831815755384869,
      "logits/chosen": -0.20445629954338074,
      "logits/rejected": -0.3627314567565918,
      "logps/chosen": -5.958117485046387,
      "logps/rejected": -38.63805389404297,
      "loss": 0.884059727191925,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5545799732208252,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.049575597047805786,
      "rewards/margins": 2.2317745685577393,
      "rewards/rejected": -2.2813501358032227,
      "step": 76,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.46368084305607826,
      "grad_norm": 2.9762485027313232,
      "learning_rate": 0.0001826238774315995,
      "logits/chosen": -0.23990976810455322,
      "logits/rejected": -0.38009804487228394,
      "logps/chosen": -5.436191082000732,
      "logps/rejected": -34.6729850769043,
      "loss": 0.7682054042816162,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4302901029586792,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.1761879026889801,
      "rewards/margins": 2.175489902496338,
      "rewards/rejected": -1.9993020296096802,
      "step": 77,
      "train_speed(iter/s)": 0.005634
    },
    {
      "epoch": 0.4697026721866767,
      "grad_norm": 2.6883339881896973,
      "learning_rate": 0.00018205796105083915,
      "logits/chosen": -0.27416834235191345,
      "logits/rejected": -0.387048602104187,
      "logps/chosen": -6.189967155456543,
      "logps/rejected": -30.830598831176758,
      "loss": 0.8499253988265991,
      "memory(GiB)": 46.13,
      "nll_loss": 0.46602901816368103,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.06820189207792282,
      "rewards/margins": 1.8341140747070312,
      "rewards/rejected": -1.7659121751785278,
      "step": 78,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.4757245013172751,
      "grad_norm": 2.6955106258392334,
      "learning_rate": 0.00018148388268569453,
      "logits/chosen": -0.22744283080101013,
      "logits/rejected": -0.35057857632637024,
      "logps/chosen": -4.485665798187256,
      "logps/rejected": -29.74733543395996,
      "loss": 0.7549667954444885,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4278162717819214,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.20732006430625916,
      "rewards/margins": 2.0227808952331543,
      "rewards/rejected": -1.8154606819152832,
      "step": 79,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.4817463304478735,
      "grad_norm": 2.907402515411377,
      "learning_rate": 0.00018090169943749476,
      "logits/chosen": -0.13104476034641266,
      "logits/rejected": -0.36177706718444824,
      "logps/chosen": -3.6902096271514893,
      "logps/rejected": -34.731773376464844,
      "loss": 0.6847038269042969,
      "memory(GiB)": 46.13,
      "nll_loss": 0.34349215030670166,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.11521060019731522,
      "rewards/margins": 1.8666913509368896,
      "rewards/rejected": -1.7514808177947998,
      "step": 80,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.48776815957847197,
      "grad_norm": 2.823246955871582,
      "learning_rate": 0.00018031146921373018,
      "logits/chosen": -0.22965751588344574,
      "logits/rejected": -0.3291424512863159,
      "logps/chosen": -5.761353969573975,
      "logps/rejected": -36.051082611083984,
      "loss": 0.7157370448112488,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4026964008808136,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.07981520146131516,
      "rewards/margins": 2.387995958328247,
      "rewards/rejected": -2.308180809020996,
      "step": 81,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.49378998870907037,
      "grad_norm": 2.73394513130188,
      "learning_rate": 0.00017971325072229226,
      "logits/chosen": -0.19317063689231873,
      "logits/rejected": -0.40784093737602234,
      "logps/chosen": -5.166616916656494,
      "logps/rejected": -40.66303634643555,
      "loss": 0.6980624198913574,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4219977855682373,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.030578728765249252,
      "rewards/margins": 2.3669514656066895,
      "rewards/rejected": -2.3363726139068604,
      "step": 82,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.4998118178396688,
      "grad_norm": 5.509173393249512,
      "learning_rate": 0.00017910710346563416,
      "logits/chosen": -0.1950196623802185,
      "logits/rejected": -0.3517180383205414,
      "logps/chosen": -8.197553634643555,
      "logps/rejected": -42.4526252746582,
      "loss": 0.9209753274917603,
      "memory(GiB)": 46.13,
      "nll_loss": 0.608447790145874,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.012901969254016876,
      "rewards/margins": 2.6168861389160156,
      "rewards/rejected": -2.6039838790893555,
      "step": 83,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5058336469702672,
      "grad_norm": 3.090348482131958,
      "learning_rate": 0.00017849308773485226,
      "logits/chosen": -0.2090476155281067,
      "logits/rejected": -0.3706829845905304,
      "logps/chosen": -10.569226264953613,
      "logps/rejected": -43.29594421386719,
      "loss": 1.0815891027450562,
      "memory(GiB)": 46.13,
      "nll_loss": 0.7449980974197388,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.03622851148247719,
      "rewards/margins": 2.7713966369628906,
      "rewards/rejected": -2.8076250553131104,
      "step": 84,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5118554761008657,
      "grad_norm": 3.75370717048645,
      "learning_rate": 0.0001778712646036894,
      "logits/chosen": -0.2378547489643097,
      "logits/rejected": -0.3873806297779083,
      "logps/chosen": -8.168487548828125,
      "logps/rejected": -37.17033386230469,
      "loss": 1.0251762866973877,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6340765953063965,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.11355286091566086,
      "rewards/margins": 2.2919938564300537,
      "rewards/rejected": -2.4055469036102295,
      "step": 85,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.517877305231464,
      "grad_norm": 3.9649507999420166,
      "learning_rate": 0.00017724169592245995,
      "logits/chosen": -0.20498239994049072,
      "logits/rejected": -0.369476854801178,
      "logps/chosen": -6.779036521911621,
      "logps/rejected": -37.98490905761719,
      "loss": 0.797798216342926,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5137072801589966,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.07833351194858551,
      "rewards/margins": 2.144557476043701,
      "rewards/rejected": -2.0662238597869873,
      "step": 86,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5238991343620625,
      "grad_norm": 3.4646809101104736,
      "learning_rate": 0.0001766044443118978,
      "logits/chosen": -0.22029685974121094,
      "logits/rejected": -0.33659541606903076,
      "logps/chosen": -5.358316421508789,
      "logps/rejected": -24.96926498413086,
      "loss": 0.809937596321106,
      "memory(GiB)": 46.13,
      "nll_loss": 0.3953189551830292,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.08655387908220291,
      "rewards/margins": 1.2862589359283447,
      "rewards/rejected": -1.1997051239013672,
      "step": 87,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5299209634926609,
      "grad_norm": 2.3174376487731934,
      "learning_rate": 0.00017595957315692782,
      "logits/chosen": -0.23393434286117554,
      "logits/rejected": -0.37657639384269714,
      "logps/chosen": -5.694438934326172,
      "logps/rejected": -26.54452896118164,
      "loss": 0.8167567253112793,
      "memory(GiB)": 46.13,
      "nll_loss": 0.46906933188438416,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.12023281306028366,
      "rewards/margins": 1.344594120979309,
      "rewards/rejected": -1.2243614196777344,
      "step": 88,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5359427926232593,
      "grad_norm": 2.29378342628479,
      "learning_rate": 0.00017530714660036112,
      "logits/chosen": -0.23060575127601624,
      "logits/rejected": -0.34717997908592224,
      "logps/chosen": -8.571242332458496,
      "logps/rejected": -27.7784423828125,
      "loss": 0.9238011837005615,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5501088500022888,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.21566246449947357,
      "rewards/margins": 1.4910237789154053,
      "rewards/rejected": -1.2753612995147705,
      "step": 89,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5419646217538577,
      "grad_norm": 4.33613395690918,
      "learning_rate": 0.00017464722953651504,
      "logits/chosen": -0.17152062058448792,
      "logits/rejected": -0.3561325669288635,
      "logps/chosen": -6.058459281921387,
      "logps/rejected": -32.55611801147461,
      "loss": 0.8752692341804504,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5144660472869873,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0842672809958458,
      "rewards/margins": 1.5552325248718262,
      "rewards/rejected": -1.4709652662277222,
      "step": 90,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5479864508844562,
      "grad_norm": 2.2437572479248047,
      "learning_rate": 0.0001739798876047584,
      "logits/chosen": -0.23511910438537598,
      "logits/rejected": -0.3851989507675171,
      "logps/chosen": -4.1521100997924805,
      "logps/rejected": -28.886817932128906,
      "loss": 0.7264631390571594,
      "memory(GiB)": 46.13,
      "nll_loss": 0.36743801832199097,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.10910709202289581,
      "rewards/margins": 1.5735148191452026,
      "rewards/rejected": -1.4644078016281128,
      "step": 91,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5540082800150545,
      "grad_norm": 2.482116460800171,
      "learning_rate": 0.00017330518718298264,
      "logits/chosen": -0.19554060697555542,
      "logits/rejected": -0.38464802503585815,
      "logps/chosen": -5.105670928955078,
      "logps/rejected": -42.76155471801758,
      "loss": 0.6233892440795898,
      "memory(GiB)": 46.13,
      "nll_loss": 0.36078184843063354,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06755763292312622,
      "rewards/margins": 2.4900288581848145,
      "rewards/rejected": -2.422471523284912,
      "step": 92,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.560030109145653,
      "grad_norm": 3.629793167114258,
      "learning_rate": 0.0001726231953809993,
      "logits/chosen": -0.23075315356254578,
      "logits/rejected": -0.4017852246761322,
      "logps/chosen": -7.799711227416992,
      "logps/rejected": -38.58491516113281,
      "loss": 0.7807697057723999,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5262271165847778,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15829423069953918,
      "rewards/margins": 2.4986987113952637,
      "rewards/rejected": -2.340404510498047,
      "step": 93,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.5660519382762514,
      "grad_norm": 10.366227149963379,
      "learning_rate": 0.0001719339800338651,
      "logits/chosen": -0.29187849164009094,
      "logits/rejected": -0.42478230595588684,
      "logps/chosen": -7.801548004150391,
      "logps/rejected": -34.29828643798828,
      "loss": 1.1007416248321533,
      "memory(GiB)": 46.13,
      "nll_loss": 0.7008671164512634,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.0017978232353925705,
      "rewards/margins": 1.9699221849441528,
      "rewards/rejected": -1.9717202186584473,
      "step": 94,
      "train_speed(iter/s)": 0.005635
    },
    {
      "epoch": 0.5720737674068498,
      "grad_norm": 4.7859039306640625,
      "learning_rate": 0.0001712376096951345,
      "logits/chosen": -0.2881244421005249,
      "logits/rejected": -0.39795249700546265,
      "logps/chosen": -6.9338459968566895,
      "logps/rejected": -34.682186126708984,
      "loss": 0.8693129420280457,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5744621157646179,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12829959392547607,
      "rewards/margins": 2.2821671962738037,
      "rewards/rejected": -2.153867244720459,
      "step": 95,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5780955965374482,
      "grad_norm": 2.521221160888672,
      "learning_rate": 0.0001705341536300409,
      "logits/chosen": -0.2661621868610382,
      "logits/rejected": -0.3923559784889221,
      "logps/chosen": -4.780788898468018,
      "logps/rejected": -28.810710906982422,
      "loss": 0.7346460819244385,
      "memory(GiB)": 46.13,
      "nll_loss": 0.40230822563171387,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.10683214664459229,
      "rewards/margins": 1.7658103704452515,
      "rewards/rejected": -1.6589782238006592,
      "step": 96,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.5841174256680467,
      "grad_norm": 5.023887634277344,
      "learning_rate": 0.00016982368180860728,
      "logits/chosen": -0.25105586647987366,
      "logits/rejected": -0.3866545855998993,
      "logps/chosen": -7.320672512054443,
      "logps/rejected": -34.95018005371094,
      "loss": 0.9987191557884216,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6328321695327759,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07585513591766357,
      "rewards/margins": 2.0516769886016846,
      "rewards/rejected": -2.127532482147217,
      "step": 97,
      "train_speed(iter/s)": 0.005636
    },
    {
      "epoch": 0.590139254798645,
      "grad_norm": 1.8328591585159302,
      "learning_rate": 0.00016910626489868649,
      "logits/chosen": -0.2209641933441162,
      "logits/rejected": -0.398262083530426,
      "logps/chosen": -4.149725437164307,
      "logps/rejected": -35.193931579589844,
      "loss": 0.6401265263557434,
      "memory(GiB)": 46.13,
      "nll_loss": 0.31758564710617065,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.14480024576187134,
      "rewards/margins": 2.074404716491699,
      "rewards/rejected": -1.9296045303344727,
      "step": 98,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.5961610839292435,
      "grad_norm": 1.696344017982483,
      "learning_rate": 0.00016838197425893202,
      "logits/chosen": -0.2796581983566284,
      "logits/rejected": -0.4003712832927704,
      "logps/chosen": -5.847466468811035,
      "logps/rejected": -30.844961166381836,
      "loss": 0.7925845384597778,
      "memory(GiB)": 46.13,
      "nll_loss": 0.4691714644432068,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15563875436782837,
      "rewards/margins": 1.9312589168548584,
      "rewards/rejected": -1.7756202220916748,
      "step": 99,
      "train_speed(iter/s)": 0.005637
    },
    {
      "epoch": 0.602182913059842,
      "grad_norm": 2.3960113525390625,
      "learning_rate": 0.00016765088193170053,
      "logits/chosen": -0.2749744653701782,
      "logits/rejected": -0.39394593238830566,
      "logps/chosen": -6.603066444396973,
      "logps/rejected": -25.912214279174805,
      "loss": 0.9519745707511902,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5554052591323853,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.11803217232227325,
      "rewards/margins": 1.5442473888397217,
      "rewards/rejected": -1.4262150526046753,
      "step": 100,
      "train_speed(iter/s)": 0.005637
    }
  ],
  "logging_steps": 1,
  "max_steps": 332,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.1504022302018765e+17,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}