{
  "best_global_step": 300,
  "best_metric": 0.60372525,
  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task1_dpo_absgt7_noidk__pari0.5_0604/v0-20250604-225329/checkpoint-300",
  "epoch": 0.6694560669456067,
  "eval_steps": 300,
  "global_step": 300,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0022315202231520223,
      "grad_norm": 11.248553276062012,
      "learning_rate": 4.444444444444445e-06,
      "logits/chosen": -0.48349103331565857,
      "logits/rejected": -0.652603030204773,
      "logps/chosen": -8.859519004821777,
      "logps/rejected": -16.016435623168945,
      "loss": 1.3468515872955322,
      "memory(GiB)": 33.34,
      "nll_loss": 0.6537042856216431,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.010678
    },
    {
      "epoch": 0.004463040446304045,
      "grad_norm": 8.97808837890625,
      "learning_rate": 8.88888888888889e-06,
      "logits/chosen": -0.5174899101257324,
      "logits/rejected": -0.6262180805206299,
      "logps/chosen": -9.287378311157227,
      "logps/rejected": -13.981985092163086,
      "loss": 1.290852665901184,
      "memory(GiB)": 36.2,
      "nll_loss": 0.5977055430412292,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.010974
    },
    {
      "epoch": 0.0066945606694560665,
      "grad_norm": 7.36937141418457,
      "learning_rate": 1.3333333333333333e-05,
      "logits/chosen": -0.42612141370773315,
      "logits/rejected": -0.6118584871292114,
      "logps/chosen": -6.56147575378418,
      "logps/rejected": -16.162681579589844,
      "loss": 1.1864153146743774,
      "memory(GiB)": 36.2,
      "nll_loss": 0.492550790309906,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": -0.0009138870518654585,
      "rewards/margins": -0.0012306026183068752,
      "rewards/rejected": 0.00031671510078012943,
      "step": 3,
      "train_speed(iter/s)": 0.011052
    },
    {
      "epoch": 0.00892608089260809,
      "grad_norm": 7.918132305145264,
      "learning_rate": 1.777777777777778e-05,
      "logits/chosen": -0.5244420170783997,
      "logits/rejected": -0.6519989371299744,
      "logps/chosen": -7.461406707763672,
      "logps/rejected": -17.006786346435547,
      "loss": 1.3234074115753174,
      "memory(GiB)": 39.28,
      "nll_loss": 0.6342031359672546,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.0032776433508843184,
      "rewards/margins": 0.008089437149465084,
      "rewards/rejected": -0.004811794031411409,
      "step": 4,
      "train_speed(iter/s)": 0.011094
    },
    {
      "epoch": 0.011157601115760111,
      "grad_norm": 9.900442123413086,
      "learning_rate": 2.2222222222222223e-05,
      "logits/chosen": -0.5073186159133911,
      "logits/rejected": -0.6049282550811768,
      "logps/chosen": -7.057607650756836,
      "logps/rejected": -15.16832447052002,
      "loss": 1.071779489517212,
      "memory(GiB)": 39.28,
      "nll_loss": 0.40041351318359375,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.022623565047979355,
      "rewards/margins": 0.04467733949422836,
      "rewards/rejected": -0.02205377072095871,
      "step": 5,
      "train_speed(iter/s)": 0.011121
    },
    {
      "epoch": 0.013389121338912133,
      "grad_norm": 5.12174129486084,
      "learning_rate": 2.6666666666666667e-05,
      "logits/chosen": -0.48164039850234985,
      "logits/rejected": -0.5844419002532959,
      "logps/chosen": -6.2985005378723145,
      "logps/rejected": -16.83810043334961,
      "loss": 1.2731884717941284,
      "memory(GiB)": 39.28,
      "nll_loss": 0.600845992565155,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.010373853147029877,
      "rewards/margins": 0.046181634068489075,
      "rewards/rejected": -0.0358077809214592,
      "step": 6,
      "train_speed(iter/s)": 0.011143
    },
    {
      "epoch": 0.015620641562064157,
      "grad_norm": 4.1249213218688965,
      "learning_rate": 3.111111111111111e-05,
      "logits/chosen": -0.5566272735595703,
      "logits/rejected": -0.6051127910614014,
      "logps/chosen": -5.424515724182129,
      "logps/rejected": -11.893548011779785,
      "loss": 1.1159045696258545,
      "memory(GiB)": 39.28,
      "nll_loss": 0.46413302421569824,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.009653191082179546,
      "rewards/margins": 0.11444520205259323,
      "rewards/rejected": -0.10479200631380081,
      "step": 7,
      "train_speed(iter/s)": 0.011163
    },
    {
      "epoch": 0.01785216178521618,
      "grad_norm": 3.8928415775299072,
      "learning_rate": 3.555555555555556e-05,
      "logits/chosen": -0.5597310662269592,
      "logits/rejected": -0.6341894268989563,
      "logps/chosen": -6.350505352020264,
      "logps/rejected": -20.239299774169922,
      "loss": 0.8606890439987183,
      "memory(GiB)": 39.28,
      "nll_loss": 0.3352409899234772,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.018700765445828438,
      "rewards/margins": 0.419380784034729,
      "rewards/rejected": -0.4380815923213959,
      "step": 8,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.0200836820083682,
      "grad_norm": 7.264344692230225,
      "learning_rate": 4e-05,
      "logits/chosen": -0.500182569026947,
      "logits/rejected": -0.595187246799469,
      "logps/chosen": -16.194358825683594,
      "logps/rejected": -20.5479793548584,
      "loss": 1.6191558837890625,
      "memory(GiB)": 39.28,
      "nll_loss": 1.0435733795166016,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.21446219086647034,
      "rewards/margins": 0.40107572078704834,
      "rewards/rejected": -0.6155378818511963,
      "step": 9,
      "train_speed(iter/s)": 0.011188
    },
    {
      "epoch": 0.022315202231520222,
      "grad_norm": 7.542423725128174,
      "learning_rate": 4.4444444444444447e-05,
      "logits/chosen": -0.41380318999290466,
      "logits/rejected": -0.5418319702148438,
      "logps/chosen": -10.30124568939209,
      "logps/rejected": -26.6314697265625,
      "loss": 1.2661710977554321,
      "memory(GiB)": 39.28,
      "nll_loss": 0.6692220568656921,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.29620301723480225,
      "rewards/margins": 0.5043031573295593,
      "rewards/rejected": -0.8005062341690063,
      "step": 10,
      "train_speed(iter/s)": 0.01119
    },
    {
      "epoch": 0.024546722454672244,
      "grad_norm": 12.055508613586426,
      "learning_rate": 4.888888888888889e-05,
      "logits/chosen": -0.45050907135009766,
      "logits/rejected": -0.5843921899795532,
      "logps/chosen": -15.530860900878906,
      "logps/rejected": -27.80389976501465,
      "loss": 1.6187087297439575,
      "memory(GiB)": 39.28,
      "nll_loss": 1.0532186031341553,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.47808465361595154,
      "rewards/margins": 0.41364309191703796,
      "rewards/rejected": -0.8917278051376343,
      "step": 11,
      "train_speed(iter/s)": 0.011193
    },
    {
      "epoch": 0.026778242677824266,
      "grad_norm": 6.556560516357422,
      "learning_rate": 5.333333333333333e-05,
      "logits/chosen": -0.4449861943721771,
      "logits/rejected": -0.5602853894233704,
      "logps/chosen": -10.715709686279297,
      "logps/rejected": -27.810680389404297,
      "loss": 1.2998032569885254,
      "memory(GiB)": 42.68,
      "nll_loss": 0.6746751070022583,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.33532828092575073,
      "rewards/margins": 0.39097556471824646,
      "rewards/rejected": -0.7263038754463196,
      "step": 12,
      "train_speed(iter/s)": 0.011193
    },
    {
      "epoch": 0.02900976290097629,
      "grad_norm": 6.815319061279297,
      "learning_rate": 5.7777777777777776e-05,
      "logits/chosen": -0.45461201667785645,
      "logits/rejected": -0.5696772933006287,
      "logps/chosen": -8.947586059570312,
      "logps/rejected": -23.452707290649414,
      "loss": 1.347914218902588,
      "memory(GiB)": 42.68,
      "nll_loss": 0.7918389439582825,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.14807622134685516,
      "rewards/margins": 0.4436675012111664,
      "rewards/rejected": -0.5917437076568604,
      "step": 13,
      "train_speed(iter/s)": 0.011196
    },
    {
      "epoch": 0.031241283124128313,
      "grad_norm": 4.015015125274658,
      "learning_rate": 6.222222222222222e-05,
      "logits/chosen": -0.4277312457561493,
      "logits/rejected": -0.5162039995193481,
      "logps/chosen": -7.859683513641357,
      "logps/rejected": -25.14496612548828,
      "loss": 1.1709389686584473,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5424311757087708,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09304673224687576,
      "rewards/margins": 0.27104437351226807,
      "rewards/rejected": -0.36409106850624084,
      "step": 14,
      "train_speed(iter/s)": 0.011191
    },
    {
      "epoch": 0.03347280334728033,
      "grad_norm": 2.5992424488067627,
      "learning_rate": 6.666666666666667e-05,
      "logits/chosen": -0.4692430794239044,
      "logits/rejected": -0.5111399292945862,
      "logps/chosen": -8.574230194091797,
      "logps/rejected": -16.565174102783203,
      "loss": 1.1323829889297485,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5113993287086487,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08413823693990707,
      "rewards/margins": 0.19407325983047485,
      "rewards/rejected": -0.27821147441864014,
      "step": 15,
      "train_speed(iter/s)": 0.011195
    },
    {
      "epoch": 0.03570432357043236,
      "grad_norm": 2.479030132293701,
      "learning_rate": 7.111111111111112e-05,
      "logits/chosen": -0.38820019364356995,
      "logits/rejected": -0.5241715908050537,
      "logps/chosen": -4.694685935974121,
      "logps/rejected": -19.62047576904297,
      "loss": 1.052955985069275,
      "memory(GiB)": 46.13,
      "nll_loss": 0.47422000765800476,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06994247436523438,
      "rewards/margins": 0.2729727327823639,
      "rewards/rejected": -0.2030302733182907,
      "step": 16,
      "train_speed(iter/s)": 0.011196
    },
    {
      "epoch": 0.03793584379358438,
      "grad_norm": 2.1875240802764893,
      "learning_rate": 7.555555555555556e-05,
      "logits/chosen": -0.4483625292778015,
      "logits/rejected": -0.5458305478096008,
      "logps/chosen": -9.014262199401855,
      "logps/rejected": -18.748149871826172,
      "loss": 1.2149709463119507,
      "memory(GiB)": 46.13,
      "nll_loss": 0.6185500621795654,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1215842068195343,
      "rewards/margins": 0.24843284487724304,
      "rewards/rejected": -0.12684863805770874,
      "step": 17,
      "train_speed(iter/s)": 0.011197
    },
    {
      "epoch": 0.0401673640167364,
      "grad_norm": 2.639186382293701,
      "learning_rate": 8e-05,
      "logits/chosen": -0.4394489824771881,
      "logits/rejected": -0.48552829027175903,
      "logps/chosen": -9.954276084899902,
      "logps/rejected": -14.681390762329102,
      "loss": 1.1648714542388916,
      "memory(GiB)": 46.13,
      "nll_loss": 0.46223676204681396,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.011617322452366352,
      "rewards/margins": 0.034979358315467834,
      "rewards/rejected": -0.02336202934384346,
      "step": 18,
      "train_speed(iter/s)": 0.011199
    },
    {
      "epoch": 0.042398884239888426,
      "grad_norm": 2.351567029953003,
      "learning_rate": 8.444444444444444e-05,
      "logits/chosen": -0.4882590174674988,
      "logits/rejected": -0.5300821661949158,
      "logps/chosen": -7.595667362213135,
      "logps/rejected": -17.29962921142578,
      "loss": 1.1524200439453125,
      "memory(GiB)": 46.13,
      "nll_loss": 0.5075826644897461,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.06894361972808838,
      "rewards/margins": 0.13017994165420532,
      "rewards/rejected": -0.061236318200826645,
      "step": 19,
      "train_speed(iter/s)": 0.011201
    },
    {
      "epoch": 0.044630404463040445,
      "grad_norm": 2.8206121921539307,
      "learning_rate": 8.888888888888889e-05,
      "logits/chosen": -0.4782502055168152,
      "logits/rejected": -0.5565172433853149,
      "logps/chosen": -6.507530212402344,
      "logps/rejected": -13.604747772216797,
      "loss": 1.089160442352295,
      "memory(GiB)": 46.13,
      "nll_loss": 0.45874831080436707,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.06730557233095169,
      "rewards/margins": 0.18410882353782654,
      "rewards/rejected": -0.11680323630571365,
      "step": 20,
      "train_speed(iter/s)": 0.011206
    },
    {
      "epoch": 0.04686192468619247,
      "grad_norm": 2.452335834503174,
      "learning_rate": 9.333333333333334e-05,
      "logits/chosen": -0.4106925129890442,
      "logits/rejected": -0.48896345496177673,
      "logps/chosen": -7.722554683685303,
      "logps/rejected": -22.094852447509766,
      "loss": 1.153997540473938,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5091268420219421,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.060057930648326874,
      "rewards/margins": 0.16068722307682037,
      "rewards/rejected": -0.1006293073296547,
      "step": 21,
      "train_speed(iter/s)": 0.011203
    },
    {
      "epoch": 0.04909344490934449,
      "grad_norm": 2.3324975967407227,
      "learning_rate": 9.777777777777778e-05,
      "logits/chosen": -0.4366399943828583,
      "logits/rejected": -0.4600733816623688,
      "logps/chosen": -9.774506568908691,
      "logps/rejected": -14.850456237792969,
      "loss": 1.185971975326538,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5413804650306702,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.08716820925474167,
      "rewards/margins": 0.17242644727230072,
      "rewards/rejected": -0.08525823801755905,
      "step": 22,
      "train_speed(iter/s)": 0.011203
    },
    {
      "epoch": 0.051324965132496514,
      "grad_norm": 2.7291500568389893,
      "learning_rate": 0.00010222222222222222,
      "logits/chosen": -0.4168204665184021,
      "logits/rejected": -0.5373830795288086,
      "logps/chosen": -7.675040245056152,
      "logps/rejected": -20.635356903076172,
      "loss": 1.1511971950531006,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4784548282623291,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.05419651046395302,
      "rewards/margins": 0.0959930419921875,
      "rewards/rejected": -0.04179652780294418,
      "step": 23,
      "train_speed(iter/s)": 0.011204
    },
    {
      "epoch": 0.05355648535564853,
      "grad_norm": 2.6357204914093018,
      "learning_rate": 0.00010666666666666667,
      "logits/chosen": -0.39612719416618347,
      "logits/rejected": -0.4547193944454193,
      "logps/chosen": -9.427398681640625,
      "logps/rejected": -20.611848831176758,
      "loss": 1.0459527969360352,
      "memory(GiB)": 46.71,
      "nll_loss": 0.42900872230529785,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.0577767938375473,
      "rewards/margins": 0.24017955362796783,
      "rewards/rejected": -0.18240275979042053,
      "step": 24,
      "train_speed(iter/s)": 0.011203
    },
    {
      "epoch": 0.05578800557880056,
      "grad_norm": 2.7458126544952393,
      "learning_rate": 0.00011111111111111112,
      "logits/chosen": -0.3978196978569031,
      "logits/rejected": -0.4898090958595276,
      "logps/chosen": -9.801487922668457,
      "logps/rejected": -21.7921199798584,
      "loss": 1.1577973365783691,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5382578372955322,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.03896612673997879,
      "rewards/margins": 0.24017201364040375,
      "rewards/rejected": -0.20120586454868317,
      "step": 25,
      "train_speed(iter/s)": 0.011205
    },
    {
      "epoch": 0.05801952580195258,
      "grad_norm": 4.185514450073242,
      "learning_rate": 0.00011555555555555555,
      "logits/chosen": -0.43978849053382874,
      "logits/rejected": -0.5358645915985107,
      "logps/chosen": -4.664649963378906,
      "logps/rejected": -18.568632125854492,
      "loss": 0.9911558032035828,
      "memory(GiB)": 46.71,
      "nll_loss": 0.39514851570129395,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.01693350076675415,
      "rewards/margins": 0.28997230529785156,
      "rewards/rejected": -0.3069058060646057,
      "step": 26,
      "train_speed(iter/s)": 0.011208
    },
    {
      "epoch": 0.0602510460251046,
      "grad_norm": 2.983151912689209,
      "learning_rate": 0.00012,
      "logits/chosen": -0.46957868337631226,
      "logits/rejected": -0.5086190700531006,
      "logps/chosen": -14.16710376739502,
      "logps/rejected": -15.19424819946289,
      "loss": 1.3119632005691528,
      "memory(GiB)": 46.71,
      "nll_loss": 0.732680082321167,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1205766424536705,
      "rewards/margins": 0.2763415277004242,
      "rewards/rejected": -0.1557648628950119,
      "step": 27,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.06248256624825663,
      "grad_norm": 3.7991929054260254,
      "learning_rate": 0.00012444444444444444,
      "logits/chosen": -0.3287929892539978,
      "logits/rejected": -0.46299535036087036,
      "logps/chosen": -6.527912616729736,
      "logps/rejected": -23.983104705810547,
      "loss": 1.0033739805221558,
      "memory(GiB)": 46.71,
      "nll_loss": 0.44460976123809814,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07841134071350098,
      "rewards/margins": 0.3698306679725647,
      "rewards/rejected": -0.29141926765441895,
      "step": 28,
      "train_speed(iter/s)": 0.011207
    },
    {
      "epoch": 0.06471408647140865,
      "grad_norm": 3.0063183307647705,
      "learning_rate": 0.00012888888888888892,
      "logits/chosen": -0.3735123574733734,
      "logits/rejected": -0.458845853805542,
      "logps/chosen": -7.325934886932373,
      "logps/rejected": -21.65030288696289,
      "loss": 1.0098869800567627,
      "memory(GiB)": 46.71,
      "nll_loss": 0.477042555809021,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.025919266045093536,
      "rewards/margins": 0.47939103841781616,
      "rewards/rejected": -0.4534717798233032,
      "step": 29,
      "train_speed(iter/s)": 0.011208
    },
    {
      "epoch": 0.06694560669456066,
      "grad_norm": 2.6957406997680664,
      "learning_rate": 0.00013333333333333334,
      "logits/chosen": -0.3790513873100281,
      "logits/rejected": -0.4815320372581482,
      "logps/chosen": -6.8280110359191895,
      "logps/rejected": -22.987621307373047,
      "loss": 1.0349267721176147,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5045362114906311,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09961891919374466,
      "rewards/margins": 0.4102502763271332,
      "rewards/rejected": -0.3106313645839691,
      "step": 30,
      "train_speed(iter/s)": 0.011209
    },
    {
      "epoch": 0.06917712691771269,
      "grad_norm": 2.7142391204833984,
      "learning_rate": 0.0001377777777777778,
      "logits/chosen": -0.36308056116104126,
      "logits/rejected": -0.44216084480285645,
      "logps/chosen": -5.850837230682373,
      "logps/rejected": -21.365463256835938,
      "loss": 0.9850523471832275,
      "memory(GiB)": 46.71,
      "nll_loss": 0.43448853492736816,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0431375615298748,
      "rewards/margins": 0.3983423113822937,
      "rewards/rejected": -0.3552047610282898,
      "step": 31,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.07140864714086471,
      "grad_norm": 3.897137403488159,
      "learning_rate": 0.00014222222222222224,
      "logits/chosen": -0.3979527950286865,
      "logits/rejected": -0.43778184056282043,
      "logps/chosen": -10.285924911499023,
      "logps/rejected": -18.7073974609375,
      "loss": 1.1233854293823242,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5153944492340088,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.026699109002947807,
      "rewards/margins": 0.23945647478103638,
      "rewards/rejected": -0.21275737881660461,
      "step": 32,
      "train_speed(iter/s)": 0.011213
    },
    {
      "epoch": 0.07364016736401674,
      "grad_norm": 3.93380069732666,
      "learning_rate": 0.00014666666666666666,
      "logits/chosen": -0.2758394777774811,
      "logits/rejected": -0.37545740604400635,
      "logps/chosen": -12.159762382507324,
      "logps/rejected": -24.926610946655273,
      "loss": 1.2487003803253174,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6138952970504761,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09166956692934036,
      "rewards/margins": 0.2808037996292114,
      "rewards/rejected": -0.3724733591079712,
      "step": 33,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.07587168758716877,
      "grad_norm": 3.21878981590271,
      "learning_rate": 0.0001511111111111111,
      "logits/chosen": -0.24420826137065887,
      "logits/rejected": -0.38765522837638855,
      "logps/chosen": -6.041399955749512,
      "logps/rejected": -22.503984451293945,
      "loss": 0.9895752668380737,
      "memory(GiB)": 46.71,
      "nll_loss": 0.44009482860565186,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.010253368876874447,
      "rewards/margins": 0.5208209753036499,
      "rewards/rejected": -0.5105676054954529,
      "step": 34,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.07810320781032078,
      "grad_norm": 5.1504316329956055,
      "learning_rate": 0.00015555555555555556,
      "logits/chosen": -0.2664889693260193,
      "logits/rejected": -0.40212178230285645,
      "logps/chosen": -5.9132609367370605,
      "logps/rejected": -24.527639389038086,
      "loss": 0.8355879187583923,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4006969928741455,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.03980417177081108,
      "rewards/margins": 0.8774286508560181,
      "rewards/rejected": -0.9172328114509583,
      "step": 35,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.0803347280334728,
      "grad_norm": 3.388197660446167,
      "learning_rate": 0.00016,
      "logits/chosen": -0.25354304909706116,
      "logits/rejected": -0.33330878615379333,
      "logps/chosen": -9.807186126708984,
      "logps/rejected": -22.963045120239258,
      "loss": 1.1024540662765503,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6454060673713684,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.001722879707813263,
      "rewards/margins": 0.8297844529151917,
      "rewards/rejected": -0.8315073847770691,
      "step": 36,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.08256624825662483,
      "grad_norm": 2.596273899078369,
      "learning_rate": 0.00016444444444444444,
      "logits/chosen": -0.21093566715717316,
      "logits/rejected": -0.3617200553417206,
      "logps/chosen": -2.8380072116851807,
      "logps/rejected": -30.42251968383789,
      "loss": 0.6882003545761108,
      "memory(GiB)": 46.71,
      "nll_loss": 0.2877930700778961,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06296033412218094,
      "rewards/margins": 1.0582321882247925,
      "rewards/rejected": -0.9952719211578369,
      "step": 37,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.08479776847977685,
      "grad_norm": 2.358452796936035,
      "learning_rate": 0.00016888888888888889,
      "logits/chosen": -0.2989475429058075,
      "logits/rejected": -0.37469911575317383,
      "logps/chosen": -6.1970534324646,
      "logps/rejected": -19.100358963012695,
      "loss": 0.8981406092643738,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3934318423271179,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.002394435927271843,
      "rewards/margins": 0.6755657196044922,
      "rewards/rejected": -0.6731712818145752,
      "step": 38,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.08702928870292886,
      "grad_norm": 2.9299051761627197,
      "learning_rate": 0.00017333333333333334,
      "logits/chosen": -0.2796667516231537,
      "logits/rejected": -0.34129300713539124,
      "logps/chosen": -8.498173713684082,
      "logps/rejected": -26.264965057373047,
      "loss": 1.012364387512207,
      "memory(GiB)": 46.71,
      "nll_loss": 0.586083173751831,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07207094132900238,
      "rewards/margins": 1.1341111660003662,
      "rewards/rejected": -1.0620403289794922,
      "step": 39,
      "train_speed(iter/s)": 0.011212
    },
    {
      "epoch": 0.08926080892608089,
      "grad_norm": 3.547189474105835,
      "learning_rate": 0.00017777777777777779,
      "logits/chosen": -0.28194305300712585,
      "logits/rejected": -0.33060967922210693,
      "logps/chosen": -7.982609748840332,
      "logps/rejected": -20.187652587890625,
      "loss": 0.9403265118598938,
      "memory(GiB)": 46.71,
      "nll_loss": 0.42783123254776,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.002833280712366104,
      "rewards/margins": 0.7690448760986328,
      "rewards/rejected": -0.7662116289138794,
      "step": 40,
      "train_speed(iter/s)": 0.011213
    },
    {
      "epoch": 0.09149232914923291,
      "grad_norm": 3.4967706203460693,
      "learning_rate": 0.00018222222222222224,
      "logits/chosen": -0.21270643174648285,
      "logits/rejected": -0.3004887104034424,
      "logps/chosen": -5.226779937744141,
      "logps/rejected": -22.208778381347656,
      "loss": 0.8620257377624512,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3914262056350708,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.018529795110225677,
      "rewards/margins": 0.9286664128303528,
      "rewards/rejected": -0.9101365804672241,
      "step": 41,
      "train_speed(iter/s)": 0.011213
    },
    {
      "epoch": 0.09372384937238494,
      "grad_norm": 6.100057601928711,
      "learning_rate": 0.0001866666666666667,
      "logits/chosen": -0.29974937438964844,
      "logits/rejected": -0.36353808641433716,
      "logps/chosen": -10.446137428283691,
      "logps/rejected": -22.973119735717773,
      "loss": 1.2477580308914185,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6882976293563843,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.2526606023311615,
      "rewards/margins": 0.5461662411689758,
      "rewards/rejected": -0.7988268136978149,
      "step": 42,
      "train_speed(iter/s)": 0.011214
    },
    {
      "epoch": 0.09595536959553697,
      "grad_norm": 4.625721454620361,
      "learning_rate": 0.00019111111111111114,
      "logits/chosen": -0.21875600516796112,
      "logits/rejected": -0.3691503405570984,
      "logps/chosen": -8.029109001159668,
      "logps/rejected": -33.986671447753906,
      "loss": 0.9927695989608765,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6190873384475708,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.10942590236663818,
      "rewards/margins": 1.7278107404708862,
      "rewards/rejected": -1.837236762046814,
      "step": 43,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.09818688981868898,
      "grad_norm": 4.994227886199951,
      "learning_rate": 0.00019555555555555556,
      "logits/chosen": -0.25192195177078247,
      "logits/rejected": -0.32522761821746826,
      "logps/chosen": -9.805305480957031,
      "logps/rejected": -34.161895751953125,
      "loss": 0.9997942447662354,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6020480990409851,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.30676746368408203,
      "rewards/margins": 1.7450387477874756,
      "rewards/rejected": -2.0518062114715576,
      "step": 44,
      "train_speed(iter/s)": 0.011214
    },
    {
      "epoch": 0.100418410041841,
      "grad_norm": 4.717008590698242,
      "learning_rate": 0.0002,
      "logits/chosen": -0.33617067337036133,
      "logits/rejected": -0.35846105217933655,
      "logps/chosen": -9.175792694091797,
      "logps/rejected": -21.536231994628906,
      "loss": 1.014683485031128,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5040097236633301,
      "rewards/accuracies": 0.625,
      "rewards/chosen": 0.014129618182778358,
      "rewards/margins": 0.7111825942993164,
      "rewards/rejected": -0.6970530152320862,
      "step": 45,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.10264993026499303,
      "grad_norm": 3.089895248413086,
      "learning_rate": 0.0001999993185874369,
      "logits/chosen": -0.3643554747104645,
      "logits/rejected": -0.3828020393848419,
      "logps/chosen": -11.385210990905762,
      "logps/rejected": -27.30405044555664,
      "loss": 0.9333115220069885,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5031883120536804,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.08416246622800827,
      "rewards/margins": 1.1319228410720825,
      "rewards/rejected": -1.216085433959961,
      "step": 46,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.10488145048814505,
      "grad_norm": 4.295483589172363,
      "learning_rate": 0.00019999727435903407,
      "logits/chosen": -0.3375833332538605,
      "logits/rejected": -0.44685542583465576,
      "logps/chosen": -4.36940336227417,
      "logps/rejected": -21.149555206298828,
      "loss": 0.959221601486206,
      "memory(GiB)": 46.71,
      "nll_loss": 0.42982059717178345,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.15477100014686584,
      "rewards/margins": 0.5941969752311707,
      "rewards/rejected": -0.4394259452819824,
      "step": 47,
      "train_speed(iter/s)": 0.011216
    },
    {
      "epoch": 0.10711297071129706,
      "grad_norm": 2.4132261276245117,
      "learning_rate": 0.0001999938673426507,
      "logits/chosen": -0.3103625178337097,
      "logits/rejected": -0.4320617616176605,
      "logps/chosen": -5.777155876159668,
      "logps/rejected": -20.897228240966797,
      "loss": 0.9094122648239136,
      "memory(GiB)": 46.71,
      "nll_loss": 0.41829684376716614,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18217116594314575,
      "rewards/margins": 0.714988112449646,
      "rewards/rejected": -0.5328169465065002,
      "step": 48,
      "train_speed(iter/s)": 0.011216
    },
    {
      "epoch": 0.10934449093444909,
      "grad_norm": 2.716282367706299,
      "learning_rate": 0.00019998909758471852,
      "logits/chosen": -0.3802068829536438,
      "logits/rejected": -0.4424472749233246,
      "logps/chosen": -7.844504356384277,
      "logps/rejected": -21.755281448364258,
      "loss": 1.0127230882644653,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5032894611358643,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.07639233767986298,
      "rewards/margins": 0.79319167137146,
      "rewards/rejected": -0.7167993187904358,
      "step": 49,
      "train_speed(iter/s)": 0.011217
    },
    {
      "epoch": 0.11157601115760112,
      "grad_norm": 3.5755205154418945,
      "learning_rate": 0.000199982965150241,
      "logits/chosen": -0.32785969972610474,
      "logits/rejected": -0.384295791387558,
      "logps/chosen": -12.152947425842285,
      "logps/rejected": -21.354625701904297,
      "loss": 1.1883673667907715,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6465156674385071,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.11994989961385727,
      "rewards/margins": 0.5093130469322205,
      "rewards/rejected": -0.6292628645896912,
      "step": 50,
      "train_speed(iter/s)": 0.011217
    },
    {
      "epoch": 0.11380753138075314,
      "grad_norm": 2.3085532188415527,
      "learning_rate": 0.00019997547012279244,
      "logits/chosen": -0.3017635941505432,
      "logits/rejected": -0.4112682342529297,
      "logps/chosen": -6.112311363220215,
      "logps/rejected": -27.533615112304688,
      "loss": 0.8378341794013977,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4290671646595001,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1803205907344818,
      "rewards/margins": 0.9729282855987549,
      "rewards/rejected": -0.7926077246665955,
      "step": 51,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.11603905160390517,
      "grad_norm": 2.699073076248169,
      "learning_rate": 0.00019996661260451704,
      "logits/chosen": -0.30682653188705444,
      "logits/rejected": -0.3987538814544678,
      "logps/chosen": -9.152729034423828,
      "logps/rejected": -34.165122985839844,
      "loss": 0.8945070505142212,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5094670653343201,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06719347834587097,
      "rewards/margins": 1.5018631219863892,
      "rewards/rejected": -1.569056510925293,
      "step": 52,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.11827057182705718,
      "grad_norm": 2.7043395042419434,
      "learning_rate": 0.00019995639271612722,
      "logits/chosen": -0.3258576989173889,
      "logits/rejected": -0.43161430954933167,
      "logps/chosen": -9.04993724822998,
      "logps/rejected": -29.138545989990234,
      "loss": 0.8318122029304504,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3884216248989105,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.015111252665519714,
      "rewards/margins": 1.0627268552780151,
      "rewards/rejected": -1.0778381824493408,
      "step": 53,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.1205020920502092,
      "grad_norm": 5.87897253036499,
      "learning_rate": 0.00019994481059690223,
      "logits/chosen": -0.36754727363586426,
      "logits/rejected": -0.4476523995399475,
      "logps/chosen": -10.40526008605957,
      "logps/rejected": -32.984188079833984,
      "loss": 1.1996067762374878,
      "memory(GiB)": 46.71,
      "nll_loss": 0.7492297887802124,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.08549878001213074,
      "rewards/margins": 1.944332242012024,
      "rewards/rejected": -2.0298309326171875,
      "step": 54,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.12273361227336123,
      "grad_norm": 2.4889438152313232,
      "learning_rate": 0.00019993186640468604,
      "logits/chosen": -0.28098922967910767,
      "logits/rejected": -0.3998360335826874,
      "logps/chosen": -11.18938159942627,
      "logps/rejected": -33.78412628173828,
      "loss": 0.97004234790802,
      "memory(GiB)": 46.71,
      "nll_loss": 0.532280445098877,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10216796398162842,
      "rewards/margins": 1.2508039474487305,
      "rewards/rejected": -1.3529717922210693,
      "step": 55,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.12496513249651325,
      "grad_norm": 3.1122195720672607,
      "learning_rate": 0.0001999175603158854,
      "logits/chosen": -0.25420328974723816,
      "logits/rejected": -0.40874406695365906,
      "logps/chosen": -9.440892219543457,
      "logps/rejected": -36.343971252441406,
      "loss": 0.8295825123786926,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4795616567134857,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.014746717177331448,
      "rewards/margins": 1.4991542100906372,
      "rewards/rejected": -1.4844075441360474,
      "step": 56,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.12719665271966527,
      "grad_norm": 3.5490951538085938,
      "learning_rate": 0.0001999018925254673,
      "logits/chosen": -0.3385506272315979,
      "logits/rejected": -0.45762842893600464,
      "logps/chosen": -9.239278793334961,
      "logps/rejected": -32.54361343383789,
      "loss": 0.9066818952560425,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5777646899223328,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03782232105731964,
      "rewards/margins": 1.4800399541854858,
      "rewards/rejected": -1.5178624391555786,
      "step": 57,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.1294281729428173,
      "grad_norm": 3.0512146949768066,
      "learning_rate": 0.00019988486324695628,
      "logits/chosen": -0.3186691403388977,
      "logits/rejected": -0.4108823835849762,
      "logps/chosen": -13.795214653015137,
      "logps/rejected": -34.02068328857422,
      "loss": 0.8977184295654297,
      "memory(GiB)": 46.71,
      "nll_loss": 0.49673789739608765,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.021573714911937714,
      "rewards/margins": 1.2825357913970947,
      "rewards/rejected": -1.3041096925735474,
      "step": 58,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.13165969316596932,
      "grad_norm": 3.224839925765991,
      "learning_rate": 0.00019986647271243163,
      "logits/chosen": -0.379732221364975,
      "logits/rejected": -0.4699477553367615,
      "logps/chosen": -6.0904154777526855,
      "logps/rejected": -22.338991165161133,
      "loss": 0.921177327632904,
      "memory(GiB)": 46.71,
      "nll_loss": 0.41462287306785583,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.054278142750263214,
      "rewards/margins": 0.6603462100028992,
      "rewards/rejected": -0.7146244049072266,
      "step": 59,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.13389121338912133,
      "grad_norm": 3.088550567626953,
      "learning_rate": 0.00019984672117252423,
      "logits/chosen": -0.3191598057746887,
      "logits/rejected": -0.4799092411994934,
      "logps/chosen": -8.916747093200684,
      "logps/rejected": -38.1026611328125,
      "loss": 0.9097569584846497,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5499361753463745,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.09366529434919357,
      "rewards/margins": 1.7305129766464233,
      "rewards/rejected": -1.6368474960327148,
      "step": 60,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.13612273361227337,
      "grad_norm": 3.0570552349090576,
      "learning_rate": 0.00019982560889641296,
      "logits/chosen": -0.34574997425079346,
      "logits/rejected": -0.5139629244804382,
      "logps/chosen": -3.300583600997925,
      "logps/rejected": -31.543636322021484,
      "loss": 0.6459969878196716,
      "memory(GiB)": 46.71,
      "nll_loss": 0.29164713621139526,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.10295584052801132,
      "rewards/margins": 1.4123960733413696,
      "rewards/rejected": -1.3094402551651,
      "step": 61,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.13835425383542538,
      "grad_norm": 4.404062747955322,
      "learning_rate": 0.00019980313617182127,
      "logits/chosen": -0.3277460038661957,
      "logits/rejected": -0.5349987745285034,
      "logps/chosen": -6.637196063995361,
      "logps/rejected": -34.88399124145508,
      "loss": 0.9588042497634888,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5974500179290771,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.10418260097503662,
      "rewards/margins": 1.5509659051895142,
      "rewards/rejected": -1.6551483869552612,
      "step": 62,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.14058577405857742,
      "grad_norm": 7.597557067871094,
      "learning_rate": 0.00019977930330501308,
      "logits/chosen": -0.40753576159477234,
      "logits/rejected": -0.5000228881835938,
      "logps/chosen": -9.73190975189209,
      "logps/rejected": -31.304035186767578,
      "loss": 1.1911669969558716,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6490563154220581,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.18217454850673676,
      "rewards/margins": 1.1403659582138062,
      "rewards/rejected": -1.322540521621704,
      "step": 63,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.14281729428172943,
      "grad_norm": 5.447345733642578,
      "learning_rate": 0.0001997541106207887,
      "logits/chosen": -0.4495885968208313,
      "logits/rejected": -0.521405816078186,
      "logps/chosen": -7.186385631561279,
      "logps/rejected": -29.914262771606445,
      "loss": 0.9763236045837402,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5955564379692078,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.02932646870613098,
      "rewards/margins": 1.4675655364990234,
      "rewards/rejected": -1.496891975402832,
      "step": 64,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.14504881450488144,
      "grad_norm": 4.937906742095947,
      "learning_rate": 0.0001997275584624803,
      "logits/chosen": -0.3159691393375397,
      "logits/rejected": -0.48153895139694214,
      "logps/chosen": -7.167924404144287,
      "logps/rejected": -33.78512954711914,
      "loss": 0.9605119228363037,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5666901469230652,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06481783092021942,
      "rewards/margins": 1.3504029512405396,
      "rewards/rejected": -1.2855849266052246,
      "step": 65,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.14728033472803348,
      "grad_norm": 4.266327857971191,
      "learning_rate": 0.00019969964719194745,
      "logits/chosen": -0.4321137070655823,
      "logits/rejected": -0.5152468681335449,
      "logps/chosen": -8.84105110168457,
      "logps/rejected": -15.863529205322266,
      "loss": 1.171226978302002,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5947294235229492,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.039367154240608215,
      "rewards/margins": 0.543604850769043,
      "rewards/rejected": -0.504237711429596,
      "step": 66,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.1495118549511855,
      "grad_norm": 3.343055009841919,
      "learning_rate": 0.0001996703771895719,
      "logits/chosen": -0.5153965353965759,
      "logits/rejected": -0.5072118043899536,
      "logps/chosen": -13.175134658813477,
      "logps/rejected": -18.902141571044922,
      "loss": 1.3753248453140259,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6952039003372192,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.00693463534116745,
      "rewards/margins": 0.5273423194885254,
      "rewards/rejected": -0.5204076766967773,
      "step": 67,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.15174337517433753,
      "grad_norm": 2.8605148792266846,
      "learning_rate": 0.00019963974885425266,
      "logits/chosen": -0.394383043050766,
      "logits/rejected": -0.4663364887237549,
      "logps/chosen": -7.570481300354004,
      "logps/rejected": -18.867816925048828,
      "loss": 1.048596739768982,
      "memory(GiB)": 46.71,
      "nll_loss": 0.447956383228302,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.06730639934539795,
      "rewards/margins": 0.42720258235931396,
      "rewards/rejected": -0.359896183013916,
      "step": 68,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.15397489539748954,
      "grad_norm": 2.4179482460021973,
      "learning_rate": 0.0001996077626034003,
      "logits/chosen": -0.43737077713012695,
      "logits/rejected": -0.49085646867752075,
      "logps/chosen": -11.430294036865234,
      "logps/rejected": -16.858463287353516,
      "loss": 1.1468638181686401,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5987557172775269,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13983695209026337,
      "rewards/margins": 0.43572792410850525,
      "rewards/rejected": -0.2958909869194031,
      "step": 69,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.15620641562064155,
      "grad_norm": 2.619527578353882,
      "learning_rate": 0.00019957441887293156,
      "logits/chosen": -0.3810432255268097,
      "logits/rejected": -0.4588402509689331,
      "logps/chosen": -7.18558931350708,
      "logps/rejected": -27.03573989868164,
      "loss": 0.9460563659667969,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4206099212169647,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.027686119079589844,
      "rewards/margins": 0.6544371843338013,
      "rewards/rejected": -0.6267510652542114,
      "step": 70,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.1584379358437936,
      "grad_norm": 2.098484754562378,
      "learning_rate": 0.0001995397181172631,
      "logits/chosen": -0.38461002707481384,
      "logits/rejected": -0.5058427453041077,
      "logps/chosen": -4.015483856201172,
      "logps/rejected": -28.747943878173828,
      "loss": 0.784480094909668,
      "memory(GiB)": 46.71,
      "nll_loss": 0.36208027601242065,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08316721022129059,
      "rewards/margins": 1.0465381145477295,
      "rewards/rejected": -0.9633709788322449,
      "step": 71,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.1606694560669456,
      "grad_norm": 2.3822543621063232,
      "learning_rate": 0.0001995036608093056,
      "logits/chosen": -0.4334784150123596,
      "logits/rejected": -0.4664575457572937,
      "logps/chosen": -8.978615760803223,
      "logps/rejected": -20.211299896240234,
      "loss": 0.8698700070381165,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3961753249168396,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03191111981868744,
      "rewards/margins": 0.8400025963783264,
      "rewards/rejected": -0.8080915808677673,
      "step": 72,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.16290097629009762,
      "grad_norm": 2.3012869358062744,
      "learning_rate": 0.00019946624744045704,
      "logits/chosen": -0.3998824954032898,
      "logits/rejected": -0.4553842544555664,
      "logps/chosen": -11.310371398925781,
      "logps/rejected": -30.51799964904785,
      "loss": 0.9168710708618164,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40945351123809814,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.09965535998344421,
      "rewards/margins": 1.1768431663513184,
      "rewards/rejected": -1.2764984369277954,
      "step": 73,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.16513249651324965,
      "grad_norm": 2.779480218887329,
      "learning_rate": 0.00019942747852059632,
      "logits/chosen": -0.3825194835662842,
      "logits/rejected": -0.4605410397052765,
      "logps/chosen": -9.17955493927002,
      "logps/rejected": -35.66179656982422,
      "loss": 0.8414367437362671,
      "memory(GiB)": 46.71,
      "nll_loss": 0.39000657200813293,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.10101831704378128,
      "rewards/margins": 1.6313400268554688,
      "rewards/rejected": -1.732358455657959,
      "step": 74,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.16736401673640167,
      "grad_norm": 2.800877571105957,
      "learning_rate": 0.00019938735457807592,
      "logits/chosen": -0.358651727437973,
      "logits/rejected": -0.43520039319992065,
      "logps/chosen": -10.250865936279297,
      "logps/rejected": -31.788986206054688,
      "loss": 0.8999710083007812,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4516286253929138,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.00027565285563468933,
      "rewards/margins": 1.1217522621154785,
      "rewards/rejected": -1.1220279932022095,
      "step": 75,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.1695955369595537,
      "grad_norm": 3.3984220027923584,
      "learning_rate": 0.00019934587615971506,
      "logits/chosen": -0.3334260582923889,
      "logits/rejected": -0.4145124852657318,
      "logps/chosen": -6.258616924285889,
      "logps/rejected": -29.351835250854492,
      "loss": 0.9294677376747131,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4573590159416199,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.005807792767882347,
      "rewards/margins": 0.9444352388381958,
      "rewards/rejected": -0.9386276006698608,
      "step": 76,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.17182705718270572,
      "grad_norm": 2.565323829650879,
      "learning_rate": 0.00019930304383079204,
      "logits/chosen": -0.3684268593788147,
      "logits/rejected": -0.4622272551059723,
      "logps/chosen": -8.288152694702148,
      "logps/rejected": -27.87519645690918,
      "loss": 0.9020035266876221,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4591652750968933,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04557492583990097,
      "rewards/margins": 1.3201135396957397,
      "rewards/rejected": -1.2745387554168701,
      "step": 77,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.17405857740585773,
      "grad_norm": 3.3687551021575928,
      "learning_rate": 0.00019925885817503663,
      "logits/chosen": -0.3821089267730713,
      "logits/rejected": -0.4754073917865753,
      "logps/chosen": -5.0986528396606445,
      "logps/rejected": -25.687240600585938,
      "loss": 0.8081955909729004,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4151981770992279,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06252564489841461,
      "rewards/margins": 1.102398157119751,
      "rewards/rejected": -1.039872407913208,
      "step": 78,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.17629009762900977,
      "grad_norm": 3.3963940143585205,
      "learning_rate": 0.00019921331979462197,
      "logits/chosen": -0.2969811260700226,
      "logits/rejected": -0.43268370628356934,
      "logps/chosen": -5.748870372772217,
      "logps/rejected": -27.06174659729004,
      "loss": 0.8259724378585815,
      "memory(GiB)": 46.71,
      "nll_loss": 0.39037975668907166,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07963411509990692,
      "rewards/margins": 0.9517691135406494,
      "rewards/rejected": -0.8721350431442261,
      "step": 79,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.17852161785216178,
      "grad_norm": 4.5683393478393555,
      "learning_rate": 0.00019916642931015662,
      "logits/chosen": -0.38007351756095886,
      "logits/rejected": -0.44829314947128296,
      "logps/chosen": -6.661276817321777,
      "logps/rejected": -23.286151885986328,
      "loss": 1.0330675840377808,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5717910528182983,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01540664117783308,
      "rewards/margins": 0.9885216951370239,
      "rewards/rejected": -0.9731149673461914,
      "step": 80,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.18075313807531382,
      "grad_norm": 2.7448818683624268,
      "learning_rate": 0.00019911818736067586,
      "logits/chosen": -0.362295538187027,
      "logits/rejected": -0.4660547375679016,
      "logps/chosen": -4.499999046325684,
      "logps/rejected": -21.89557647705078,
      "loss": 0.7531627416610718,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40978550910949707,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23986083269119263,
      "rewards/margins": 1.2353352308273315,
      "rewards/rejected": -0.9954744577407837,
      "step": 81,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.18298465829846583,
      "grad_norm": 2.189898729324341,
      "learning_rate": 0.00019906859460363307,
      "logits/chosen": -0.34852471947669983,
      "logits/rejected": -0.4147579073905945,
      "logps/chosen": -7.054965972900391,
      "logps/rejected": -27.020904541015625,
      "loss": 0.6932824850082397,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40176427364349365,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19743630290031433,
      "rewards/margins": 1.7189998626708984,
      "rewards/rejected": -1.5215637683868408,
      "step": 82,
      "train_speed(iter/s)": 0.011183
    },
    {
      "epoch": 0.18521617852161784,
      "grad_norm": 6.675446510314941,
      "learning_rate": 0.00019901765171489086,
      "logits/chosen": -0.2617488205432892,
      "logits/rejected": -0.42673808336257935,
      "logps/chosen": -10.860867500305176,
      "logps/rejected": -47.94599533081055,
      "loss": 1.1189333200454712,
      "memory(GiB)": 46.71,
      "nll_loss": 0.7764029502868652,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.33700618147850037,
      "rewards/margins": 2.8077898025512695,
      "rewards/rejected": -3.144796133041382,
      "step": 83,
      "train_speed(iter/s)": 0.011184
    },
    {
      "epoch": 0.18744769874476988,
      "grad_norm": 3.3365304470062256,
      "learning_rate": 0.0001989653593887117,
      "logits/chosen": -0.33305802941322327,
      "logits/rejected": -0.41463255882263184,
      "logps/chosen": -6.076395034790039,
      "logps/rejected": -32.26319885253906,
      "loss": 0.7819843292236328,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4539300501346588,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10270927101373672,
      "rewards/margins": 2.0022926330566406,
      "rewards/rejected": -1.899583339691162,
      "step": 84,
      "train_speed(iter/s)": 0.011185
    },
    {
      "epoch": 0.1896792189679219,
      "grad_norm": 3.774372100830078,
      "learning_rate": 0.00019891171833774854,
      "logits/chosen": -0.2869430482387543,
      "logits/rejected": -0.38458919525146484,
      "logps/chosen": -9.53564453125,
      "logps/rejected": -44.279476165771484,
      "loss": 0.9625971913337708,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5546936392784119,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.1114846020936966,
      "rewards/margins": 2.2966198921203613,
      "rewards/rejected": -2.408104419708252,
      "step": 85,
      "train_speed(iter/s)": 0.011185
    },
    {
      "epoch": 0.19191073919107393,
      "grad_norm": 7.161458969116211,
      "learning_rate": 0.00019885672929303508,
      "logits/chosen": -0.2910900115966797,
      "logits/rejected": -0.4121524691581726,
      "logps/chosen": -10.86208724975586,
      "logps/rejected": -43.04856491088867,
      "loss": 0.8533170819282532,
      "memory(GiB)": 46.71,
      "nll_loss": 0.49533799290657043,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0742935836315155,
      "rewards/margins": 2.4323437213897705,
      "rewards/rejected": -2.5066375732421875,
      "step": 86,
      "train_speed(iter/s)": 0.011185
    },
    {
      "epoch": 0.19414225941422594,
      "grad_norm": 3.097945213317871,
      "learning_rate": 0.00019880039300397591,
      "logits/chosen": -0.38776522874832153,
      "logits/rejected": -0.4260600805282593,
      "logps/chosen": -7.062775135040283,
      "logps/rejected": -30.766754150390625,
      "loss": 0.8838493227958679,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5182939767837524,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07534455507993698,
      "rewards/margins": 1.623365044593811,
      "rewards/rejected": -1.5480204820632935,
      "step": 87,
      "train_speed(iter/s)": 0.011186
    },
    {
      "epoch": 0.19637377963737795,
      "grad_norm": 2.158613443374634,
      "learning_rate": 0.00019874271023833604,
      "logits/chosen": -0.3456321060657501,
      "logits/rejected": -0.43574386835098267,
      "logps/chosen": -7.558653831481934,
      "logps/rejected": -30.61342430114746,
      "loss": 0.7866738438606262,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4206286072731018,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15493661165237427,
      "rewards/margins": 1.5608376264572144,
      "rewards/rejected": -1.4059010744094849,
      "step": 88,
      "train_speed(iter/s)": 0.011187
    },
    {
      "epoch": 0.19860529986053,
      "grad_norm": 3.826850414276123,
      "learning_rate": 0.00019868368178223075,
      "logits/chosen": -0.3609853982925415,
      "logits/rejected": -0.4561440646648407,
      "logps/chosen": -9.480233192443848,
      "logps/rejected": -24.056900024414062,
      "loss": 1.0715264081954956,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5571523904800415,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.019891131669282913,
      "rewards/margins": 0.8333019018173218,
      "rewards/rejected": -0.8531929850578308,
      "step": 89,
      "train_speed(iter/s)": 0.011188
    },
    {
      "epoch": 0.200836820083682,
      "grad_norm": 2.5776290893554688,
      "learning_rate": 0.00019862330844011466,
      "logits/chosen": -0.39898231625556946,
      "logits/rejected": -0.5255773663520813,
      "logps/chosen": -6.868825435638428,
      "logps/rejected": -27.24391746520996,
      "loss": 0.9221024513244629,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5272523760795593,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.16320021450519562,
      "rewards/margins": 1.3196171522140503,
      "rewards/rejected": -1.156416893005371,
      "step": 90,
      "train_speed(iter/s)": 0.011188
    },
    {
      "epoch": 0.20306834030683404,
      "grad_norm": 2.8630950450897217,
      "learning_rate": 0.00019856159103477086,
      "logits/chosen": -0.38147053122520447,
      "logits/rejected": -0.49192503094673157,
      "logps/chosen": -4.319600582122803,
      "logps/rejected": -29.042064666748047,
      "loss": 0.6743653416633606,
      "memory(GiB)": 46.71,
      "nll_loss": 0.2815118730068207,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06610685586929321,
      "rewards/margins": 1.2929189205169678,
      "rewards/rejected": -1.2268121242523193,
      "step": 91,
      "train_speed(iter/s)": 0.011189
    },
    {
      "epoch": 0.20529986052998606,
      "grad_norm": 2.3517754077911377,
      "learning_rate": 0.00019849853040729962,
      "logits/chosen": -0.3899337947368622,
      "logits/rejected": -0.512879490852356,
      "logps/chosen": -3.6735916137695312,
      "logps/rejected": -38.82426452636719,
      "loss": 0.575823962688446,
      "memory(GiB)": 46.71,
      "nll_loss": 0.27289441227912903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.014910156838595867,
      "rewards/margins": 2.080000638961792,
      "rewards/rejected": -2.0650904178619385,
      "step": 92,
      "train_speed(iter/s)": 0.011189
    },
    {
      "epoch": 0.20753138075313807,
      "grad_norm": 7.1787543296813965,
      "learning_rate": 0.00019843412741710705,
      "logits/chosen": -0.4152039885520935,
      "logits/rejected": -0.5459296107292175,
      "logps/chosen": -9.160662651062012,
      "logps/rejected": -33.41329574584961,
      "loss": 0.9777874946594238,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5870996713638306,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.06245291978120804,
      "rewards/margins": 1.6734349727630615,
      "rewards/rejected": -1.7358877658843994,
      "step": 93,
      "train_speed(iter/s)": 0.01119
    },
    {
      "epoch": 0.2097629009762901,
      "grad_norm": 6.694753646850586,
      "learning_rate": 0.00019836838294189327,
      "logits/chosen": -0.2928682565689087,
      "logits/rejected": -0.5464470386505127,
      "logps/chosen": -4.742891788482666,
      "logps/rejected": -48.63407516479492,
      "loss": 0.6743770241737366,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4215732514858246,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.016849342733621597,
      "rewards/margins": 2.5316309928894043,
      "rewards/rejected": -2.548480272293091,
      "step": 94,
      "train_speed(iter/s)": 0.011189
    },
    {
      "epoch": 0.21199442119944212,
      "grad_norm": 3.5523533821105957,
      "learning_rate": 0.00019830129787764048,
      "logits/chosen": -0.43435996770858765,
      "logits/rejected": -0.5884309411048889,
      "logps/chosen": -7.528013229370117,
      "logps/rejected": -40.399600982666016,
      "loss": 0.655762255191803,
      "memory(GiB)": 46.71,
      "nll_loss": 0.36153632402420044,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.16746671497821808,
      "rewards/margins": 2.1469664573669434,
      "rewards/rejected": -2.3144330978393555,
      "step": 95,
      "train_speed(iter/s)": 0.011189
    },
    {
      "epoch": 0.21422594142259413,
      "grad_norm": 3.652083396911621,
      "learning_rate": 0.00019823287313860087,
      "logits/chosen": -0.39443954825401306,
      "logits/rejected": -0.5311736464500427,
      "logps/chosen": -10.99143123626709,
      "logps/rejected": -34.9860725402832,
      "loss": 1.1298080682754517,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6828101873397827,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.050042107701301575,
      "rewards/margins": 1.6755130290985107,
      "rewards/rejected": -1.725555181503296,
      "step": 96,
      "train_speed(iter/s)": 0.011189
    },
    {
      "epoch": 0.21645746164574617,
      "grad_norm": 3.373784065246582,
      "learning_rate": 0.00019816310965728388,
      "logits/chosen": -0.4609702229499817,
      "logits/rejected": -0.534326434135437,
      "logps/chosen": -9.729392051696777,
      "logps/rejected": -26.942209243774414,
      "loss": 0.970382571220398,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4684171676635742,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.06363824754953384,
      "rewards/margins": 1.244420051574707,
      "rewards/rejected": -1.308058261871338,
      "step": 97,
      "train_speed(iter/s)": 0.01119
    },
    {
      "epoch": 0.21868898186889818,
      "grad_norm": 2.9718940258026123,
      "learning_rate": 0.00019809200838444383,
      "logits/chosen": -0.4923046827316284,
      "logits/rejected": -0.5684210062026978,
      "logps/chosen": -7.5212883949279785,
      "logps/rejected": -26.84278678894043,
      "loss": 0.7905946373939514,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4011859893798828,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1688103973865509,
      "rewards/margins": 1.1646181344985962,
      "rewards/rejected": -0.9958077669143677,
      "step": 98,
      "train_speed(iter/s)": 0.011191
    },
    {
      "epoch": 0.22092050209205022,
      "grad_norm": 4.091674327850342,
      "learning_rate": 0.0001980195702890667,
      "logits/chosen": -0.41893988847732544,
      "logits/rejected": -0.5414128303527832,
      "logps/chosen": -7.620563507080078,
      "logps/rejected": -34.5891227722168,
      "loss": 0.8988033533096313,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4130387604236603,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0692744329571724,
      "rewards/margins": 1.639555811882019,
      "rewards/rejected": -1.7088301181793213,
      "step": 99,
      "train_speed(iter/s)": 0.011191
    },
    {
      "epoch": 0.22315202231520223,
      "grad_norm": 4.4936113357543945,
      "learning_rate": 0.00019794579635835704,
      "logits/chosen": -0.4513593912124634,
      "logits/rejected": -0.5386379361152649,
      "logps/chosen": -7.520744323730469,
      "logps/rejected": -21.09442710876465,
      "loss": 1.0648761987686157,
      "memory(GiB)": 46.71,
      "nll_loss": 0.551341712474823,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.06751161813735962,
      "rewards/margins": 0.5798344016075134,
      "rewards/rejected": -0.5123227834701538,
      "step": 100,
      "train_speed(iter/s)": 0.011192
    },
    {
      "epoch": 0.22538354253835424,
      "grad_norm": 3.2135934829711914,
      "learning_rate": 0.00019787068759772458,
      "logits/chosen": -0.30221307277679443,
      "logits/rejected": -0.4815448820590973,
      "logps/chosen": -5.885772705078125,
      "logps/rejected": -42.119300842285156,
      "loss": 0.7016221880912781,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3754323422908783,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.016318581998348236,
      "rewards/margins": 1.9625664949417114,
      "rewards/rejected": -1.9462475776672363,
      "step": 101,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.22761506276150628,
      "grad_norm": 2.6394968032836914,
      "learning_rate": 0.0001977942450307703,
      "logits/chosen": -0.4444178342819214,
      "logits/rejected": -0.550942599773407,
      "logps/chosen": -2.515904188156128,
      "logps/rejected": -24.57438087463379,
      "loss": 0.5894904136657715,
      "memory(GiB)": 46.71,
      "nll_loss": 0.2374342381954193,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13210514187812805,
      "rewards/margins": 1.279597520828247,
      "rewards/rejected": -1.1474921703338623,
      "step": 102,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.2298465829846583,
      "grad_norm": 3.511199712753296,
      "learning_rate": 0.00019771646969927276,
      "logits/chosen": -0.37454891204833984,
      "logits/rejected": -0.5178055763244629,
      "logps/chosen": -11.968851089477539,
      "logps/rejected": -43.35523986816406,
      "loss": 0.9422297477722168,
      "memory(GiB)": 46.71,
      "nll_loss": 0.643075704574585,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.14428094029426575,
      "rewards/margins": 2.009899377822876,
      "rewards/rejected": -2.1541800498962402,
      "step": 103,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.23207810320781033,
      "grad_norm": 3.9855709075927734,
      "learning_rate": 0.00019763736266317373,
      "logits/chosen": -0.4065098166465759,
      "logits/rejected": -0.4758867025375366,
      "logps/chosen": -10.250575065612793,
      "logps/rejected": -30.894758224487305,
      "loss": 1.0436267852783203,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6723781228065491,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.054463766515254974,
      "rewards/margins": 1.513323187828064,
      "rewards/rejected": -1.4588593244552612,
      "step": 104,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.23430962343096234,
      "grad_norm": 2.9467523097991943,
      "learning_rate": 0.00019755692500056377,
      "logits/chosen": -0.3900934159755707,
      "logits/rejected": -0.47925370931625366,
      "logps/chosen": -7.484468460083008,
      "logps/rejected": -43.62319564819336,
      "loss": 0.7645498514175415,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4648410677909851,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10762156546115875,
      "rewards/margins": 2.406116247177124,
      "rewards/rejected": -2.513737916946411,
      "step": 105,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.23654114365411436,
      "grad_norm": 4.7432637214660645,
      "learning_rate": 0.0001974751578076675,
      "logits/chosen": -0.4480246603488922,
      "logits/rejected": -0.5329591631889343,
      "logps/chosen": -8.911030769348145,
      "logps/rejected": -36.37174987792969,
      "loss": 1.0600045919418335,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6955873966217041,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.00720573216676712,
      "rewards/margins": 2.03155517578125,
      "rewards/rejected": -2.0243494510650635,
      "step": 106,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.2387726638772664,
      "grad_norm": 5.058660984039307,
      "learning_rate": 0.0001973920621988288,
      "logits/chosen": -0.3936890661716461,
      "logits/rejected": -0.4737499952316284,
      "logps/chosen": -9.114482879638672,
      "logps/rejected": -38.9593505859375,
      "loss": 0.9397415518760681,
      "memory(GiB)": 46.71,
      "nll_loss": 0.619577169418335,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.1711510568857193,
      "rewards/margins": 2.256584882736206,
      "rewards/rejected": -2.4277358055114746,
      "step": 107,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.2410041841004184,
      "grad_norm": 5.576498985290527,
      "learning_rate": 0.00019730763930649557,
      "logits/chosen": -0.3989205062389374,
      "logits/rejected": -0.5156173706054688,
      "logps/chosen": -7.493094444274902,
      "logps/rejected": -38.66194152832031,
      "loss": 1.1445399522781372,
      "memory(GiB)": 46.71,
      "nll_loss": 0.7500584125518799,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.078619584441185,
      "rewards/margins": 2.2690536975860596,
      "rewards/rejected": -2.347673177719116,
      "step": 108,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.24323570432357045,
      "grad_norm": 5.548501491546631,
      "learning_rate": 0.0001972218902812041,
      "logits/chosen": -0.4444655776023865,
      "logits/rejected": -0.5106954574584961,
      "logps/chosen": -10.32758903503418,
      "logps/rejected": -29.883926391601562,
      "loss": 1.1218242645263672,
      "memory(GiB)": 46.71,
      "nll_loss": 0.7221024036407471,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.10507391393184662,
      "rewards/margins": 1.7042087316513062,
      "rewards/rejected": -1.8092825412750244,
      "step": 109,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.24546722454672246,
      "grad_norm": 4.309199333190918,
      "learning_rate": 0.0001971348162915637,
      "logits/chosen": -0.3248450756072998,
      "logits/rejected": -0.4501338601112366,
      "logps/chosen": -12.836463928222656,
      "logps/rejected": -46.953514099121094,
      "loss": 1.0559560060501099,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6316272020339966,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07269488275051117,
      "rewards/margins": 1.5467530488967896,
      "rewards/rejected": -1.6194478273391724,
      "step": 110,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.24769874476987447,
      "grad_norm": 3.218020439147949,
      "learning_rate": 0.0001970464185242406,
      "logits/chosen": -0.39539581537246704,
      "logits/rejected": -0.48628556728363037,
      "logps/chosen": -8.726967811584473,
      "logps/rejected": -29.990245819091797,
      "loss": 0.9488601088523865,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6031092405319214,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.023079559206962585,
      "rewards/margins": 1.3200984001159668,
      "rewards/rejected": -1.3431780338287354,
      "step": 111,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.2499302649930265,
      "grad_norm": 2.6869122982025146,
      "learning_rate": 0.00019695669818394176,
      "logits/chosen": -0.41482657194137573,
      "logits/rejected": -0.4805757999420166,
      "logps/chosen": -6.431066036224365,
      "logps/rejected": -25.98788833618164,
      "loss": 0.8483121991157532,
      "memory(GiB)": 46.71,
      "nll_loss": 0.37977075576782227,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.08659300208091736,
      "rewards/margins": 0.9510751962661743,
      "rewards/rejected": -0.8644822835922241,
      "step": 112,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.25216178521617855,
      "grad_norm": 2.551693916320801,
      "learning_rate": 0.00019686565649339852,
      "logits/chosen": -0.3926604092121124,
      "logits/rejected": -0.4559895694255829,
      "logps/chosen": -7.078885078430176,
      "logps/rejected": -27.05070686340332,
      "loss": 0.8254140019416809,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4410409927368164,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.20977970957756042,
      "rewards/margins": 1.3954874277114868,
      "rewards/rejected": -1.1857078075408936,
      "step": 113,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.25439330543933053,
      "grad_norm": 2.873304843902588,
      "learning_rate": 0.0001967732946933499,
      "logits/chosen": -0.40831753611564636,
      "logits/rejected": -0.43265292048454285,
      "logps/chosen": -9.373550415039062,
      "logps/rejected": -18.696136474609375,
      "loss": 1.1380702257156372,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5557818412780762,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.030941195785999298,
      "rewards/margins": 0.6564021110534668,
      "rewards/rejected": -0.6873433589935303,
      "step": 114,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.25662482566248257,
      "grad_norm": 2.8062174320220947,
      "learning_rate": 0.00019667961404252573,
      "logits/chosen": -0.39943259954452515,
      "logits/rejected": -0.4698421061038971,
      "logps/chosen": -6.604292869567871,
      "logps/rejected": -27.13275909423828,
      "loss": 0.904656708240509,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40879639983177185,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06587034463882446,
      "rewards/margins": 1.2184947729110718,
      "rewards/rejected": -1.1526243686676025,
      "step": 115,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.2588563458856346,
      "grad_norm": 2.269819974899292,
      "learning_rate": 0.00019658461581762948,
      "logits/chosen": -0.37201863527297974,
      "logits/rejected": -0.49889877438545227,
      "logps/chosen": -5.184394836425781,
      "logps/rejected": -33.83866500854492,
      "loss": 0.6697641015052795,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3163735568523407,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.12605488300323486,
      "rewards/margins": 1.7490179538726807,
      "rewards/rejected": -1.6229631900787354,
      "step": 116,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.2610878661087866,
      "grad_norm": 2.150320291519165,
      "learning_rate": 0.0001964883013133208,
      "logits/chosen": -0.3641276955604553,
      "logits/rejected": -0.46924924850463867,
      "logps/chosen": -6.894084930419922,
      "logps/rejected": -30.72964096069336,
      "loss": 0.7019263505935669,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3518219292163849,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11152325570583344,
      "rewards/margins": 1.5766544342041016,
      "rewards/rejected": -1.465131163597107,
      "step": 117,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.26331938633193863,
      "grad_norm": 1.940995693206787,
      "learning_rate": 0.00019639067184219796,
      "logits/chosen": -0.4233744740486145,
      "logits/rejected": -0.49646085500717163,
      "logps/chosen": -2.5980770587921143,
      "logps/rejected": -22.459583282470703,
      "loss": 0.5635585784912109,
      "memory(GiB)": 46.71,
      "nll_loss": 0.23038321733474731,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20914426445960999,
      "rewards/margins": 1.3377940654754639,
      "rewards/rejected": -1.1286499500274658,
      "step": 118,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.26555090655509067,
      "grad_norm": 4.13163423538208,
      "learning_rate": 0.00019629172873477995,
      "logits/chosen": -0.43275851011276245,
      "logits/rejected": -0.5510309934616089,
      "logps/chosen": -7.118757724761963,
      "logps/rejected": -33.80887222290039,
      "loss": 0.9356746077537537,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5518943071365356,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06265924125909805,
      "rewards/margins": 1.4980642795562744,
      "rewards/rejected": -1.5607234239578247,
      "step": 119,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.26778242677824265,
      "grad_norm": 4.10536003112793,
      "learning_rate": 0.00019619147333948823,
      "logits/chosen": -0.3524538576602936,
      "logits/rejected": -0.5940331220626831,
      "logps/chosen": -3.5720577239990234,
      "logps/rejected": -37.1466064453125,
      "loss": 0.6390153169631958,
      "memory(GiB)": 46.71,
      "nll_loss": 0.33055761456489563,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.012503441423177719,
      "rewards/margins": 1.8064663410186768,
      "rewards/rejected": -1.818969964981079,
      "step": 120,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.2700139470013947,
      "grad_norm": 3.469205856323242,
      "learning_rate": 0.00019608990702262862,
      "logits/chosen": -0.43667861819267273,
      "logits/rejected": -0.5170078873634338,
      "logps/chosen": -10.792959213256836,
      "logps/rejected": -33.0848503112793,
      "loss": 0.915253758430481,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5493744015693665,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0627615749835968,
      "rewards/margins": 1.9415191411972046,
      "rewards/rejected": -2.0042805671691895,
      "step": 121,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.27224546722454673,
      "grad_norm": 3.509106159210205,
      "learning_rate": 0.0001959870311683723,
      "logits/chosen": -0.42921414971351624,
      "logits/rejected": -0.5258873105049133,
      "logps/chosen": -11.711372375488281,
      "logps/rejected": -33.04314041137695,
      "loss": 0.928779125213623,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5003984570503235,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.20121777057647705,
      "rewards/margins": 1.2945905923843384,
      "rewards/rejected": -1.4958083629608154,
      "step": 122,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.2744769874476987,
      "grad_norm": 3.868866443634033,
      "learning_rate": 0.00019588284717873738,
      "logits/chosen": -0.47573891282081604,
      "logits/rejected": -0.5573767423629761,
      "logps/chosen": -11.404234886169434,
      "logps/rejected": -44.85041427612305,
      "loss": 0.868504524230957,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5618444085121155,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06724707782268524,
      "rewards/margins": 2.6722822189331055,
      "rewards/rejected": -2.7395291328430176,
      "step": 123,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.27670850767085076,
      "grad_norm": 2.392876625061035,
      "learning_rate": 0.00019577735647356928,
      "logits/chosen": -0.4367309510707855,
      "logits/rejected": -0.5319121479988098,
      "logps/chosen": -6.822817802429199,
      "logps/rejected": -31.786712646484375,
      "loss": 0.7306314706802368,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40483999252319336,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14910897612571716,
      "rewards/margins": 1.7431702613830566,
      "rewards/rejected": -1.594061255455017,
      "step": 124,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.2789400278940028,
      "grad_norm": 3.1055872440338135,
      "learning_rate": 0.00019567056049052203,
      "logits/chosen": -0.34188365936279297,
      "logits/rejected": -0.5163190364837646,
      "logps/chosen": -4.810876369476318,
      "logps/rejected": -44.724796295166016,
      "loss": 0.5886633396148682,
      "memory(GiB)": 46.71,
      "nll_loss": 0.33786720037460327,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.056971918791532516,
      "rewards/margins": 2.44872784614563,
      "rewards/rejected": -2.391756296157837,
      "step": 125,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.28117154811715483,
      "grad_norm": 5.503366947174072,
      "learning_rate": 0.00019556246068503795,
      "logits/chosen": -0.46696051955223083,
      "logits/rejected": -0.5219815969467163,
      "logps/chosen": -12.945513725280762,
      "logps/rejected": -39.0778923034668,
      "loss": 0.9357361197471619,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6219462752342224,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.06543703377246857,
      "rewards/margins": 2.5177855491638184,
      "rewards/rejected": -2.5832223892211914,
      "step": 126,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.2834030683403068,
      "grad_norm": 4.39600133895874,
      "learning_rate": 0.00019545305853032848,
      "logits/chosen": -0.40930306911468506,
      "logits/rejected": -0.5165880918502808,
      "logps/chosen": -8.137343406677246,
      "logps/rejected": -36.05834197998047,
      "loss": 0.9704120755195618,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6009897589683533,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.06638351082801819,
      "rewards/margins": 1.870111107826233,
      "rewards/rejected": -1.9364944696426392,
      "step": 127,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.28563458856345886,
      "grad_norm": 2.2071218490600586,
      "learning_rate": 0.0001953423555173536,
      "logits/chosen": -0.3733062446117401,
      "logits/rejected": -0.5163143277168274,
      "logps/chosen": -5.431221008300781,
      "logps/rejected": -50.760562896728516,
      "loss": 0.5296469926834106,
      "memory(GiB)": 46.71,
      "nll_loss": 0.33633342385292053,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02796844020485878,
      "rewards/margins": 2.690657377243042,
      "rewards/rejected": -2.662688732147217,
      "step": 128,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.2878661087866109,
      "grad_norm": 3.5191235542297363,
      "learning_rate": 0.0001952303531548018,
      "logits/chosen": -0.36135807633399963,
      "logits/rejected": -0.46837830543518066,
      "logps/chosen": -5.7369561195373535,
      "logps/rejected": -40.44853591918945,
      "loss": 0.591960072517395,
      "memory(GiB)": 46.71,
      "nll_loss": 0.2807152271270752,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08742895722389221,
      "rewards/margins": 2.5908262729644775,
      "rewards/rejected": -2.503397226333618,
      "step": 129,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.2900976290097629,
      "grad_norm": 3.575482130050659,
      "learning_rate": 0.00019511705296906945,
      "logits/chosen": -0.3611457347869873,
      "logits/rejected": -0.503516435623169,
      "logps/chosen": -5.223241329193115,
      "logps/rejected": -37.931541442871094,
      "loss": 0.534879207611084,
      "memory(GiB)": 46.71,
      "nll_loss": 0.29983922839164734,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0478663444519043,
      "rewards/margins": 2.279771327972412,
      "rewards/rejected": -2.231905221939087,
      "step": 130,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.2923291492329149,
      "grad_norm": 3.9460039138793945,
      "learning_rate": 0.00019500245650423987,
      "logits/chosen": -0.3660318851470947,
      "logits/rejected": -0.4796612560749054,
      "logps/chosen": -11.665671348571777,
      "logps/rejected": -49.753868103027344,
      "loss": 0.9670472145080566,
      "memory(GiB)": 46.71,
      "nll_loss": 0.7097752690315247,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.1927420198917389,
      "rewards/margins": 3.0559210777282715,
      "rewards/rejected": -3.2486634254455566,
      "step": 131,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.29456066945606696,
      "grad_norm": 6.069462776184082,
      "learning_rate": 0.00019488656532206256,
      "logits/chosen": -0.32610565423965454,
      "logits/rejected": -0.4974173605442047,
      "logps/chosen": -6.3772382736206055,
      "logps/rejected": -48.19947814941406,
      "loss": 0.575228214263916,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3116891086101532,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06776230037212372,
      "rewards/margins": 2.7837750911712646,
      "rewards/rejected": -2.8515372276306152,
      "step": 132,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.29679218967921894,
      "grad_norm": 8.746431350708008,
      "learning_rate": 0.00019476938100193166,
      "logits/chosen": -0.3666072189807892,
      "logits/rejected": -0.4251011908054352,
      "logps/chosen": -11.349700927734375,
      "logps/rejected": -45.43285369873047,
      "loss": 0.9895598292350769,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6815392971038818,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.039151158183813095,
      "rewards/margins": 2.9056763648986816,
      "rewards/rejected": -2.9448275566101074,
      "step": 133,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.299023709902371,
      "grad_norm": 2.3924574851989746,
      "learning_rate": 0.00019465090514086448,
      "logits/chosen": -0.3149105906486511,
      "logits/rejected": -0.4992123246192932,
      "logps/chosen": -6.0071845054626465,
      "logps/rejected": -48.10839080810547,
      "loss": 0.524428129196167,
      "memory(GiB)": 46.71,
      "nll_loss": 0.34304654598236084,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.037540286779403687,
      "rewards/margins": 2.7853667736053467,
      "rewards/rejected": -2.74782657623291,
      "step": 134,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.301255230125523,
      "grad_norm": 5.539307594299316,
      "learning_rate": 0.00019453113935347983,
      "logits/chosen": -0.31063827872276306,
      "logits/rejected": -0.4415431618690491,
      "logps/chosen": -9.355257987976074,
      "logps/rejected": -32.0480842590332,
      "loss": 0.9339735507965088,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5960800647735596,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.02493138611316681,
      "rewards/margins": 1.7281333208084106,
      "rewards/rejected": -1.753064751625061,
      "step": 135,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.30348675034867506,
      "grad_norm": 2.773390531539917,
      "learning_rate": 0.000194410085271976,
      "logits/chosen": -0.32995009422302246,
      "logits/rejected": -0.47322651743888855,
      "logps/chosen": -7.31096076965332,
      "logps/rejected": -39.725521087646484,
      "loss": 0.7643193006515503,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4867798089981079,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06533154845237732,
      "rewards/margins": 2.221080780029297,
      "rewards/rejected": -2.1557490825653076,
      "step": 136,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.30571827057182704,
      "grad_norm": 3.071211099624634,
      "learning_rate": 0.00019428774454610843,
      "logits/chosen": -0.3505553901195526,
      "logits/rejected": -0.45163339376449585,
      "logps/chosen": -7.793257713317871,
      "logps/rejected": -34.78621292114258,
      "loss": 0.726679265499115,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4261872172355652,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06730077415704727,
      "rewards/margins": 2.1191506385803223,
      "rewards/rejected": -2.0518500804901123,
      "step": 137,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.3079497907949791,
      "grad_norm": 2.8342206478118896,
      "learning_rate": 0.00019416411884316722,
      "logits/chosen": -0.23897941410541534,
      "logits/rejected": -0.4108068645000458,
      "logps/chosen": -5.271795272827148,
      "logps/rejected": -33.89106750488281,
      "loss": 0.6566328406333923,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3124980330467224,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06509758532047272,
      "rewards/margins": 1.5957400798797607,
      "rewards/rejected": -1.5306426286697388,
      "step": 138,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.3101813110181311,
      "grad_norm": 2.3552298545837402,
      "learning_rate": 0.00019403920984795454,
      "logits/chosen": -0.22743001580238342,
      "logits/rejected": -0.3986920714378357,
      "logps/chosen": -4.359319686889648,
      "logps/rejected": -34.49266815185547,
      "loss": 0.7009214162826538,
      "memory(GiB)": 46.71,
      "nll_loss": 0.37971875071525574,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1354263424873352,
      "rewards/margins": 1.9385743141174316,
      "rewards/rejected": -1.8031479120254517,
      "step": 139,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.3124128312412831,
      "grad_norm": 3.9349329471588135,
      "learning_rate": 0.00019391301926276156,
      "logits/chosen": -0.25204330682754517,
      "logits/rejected": -0.40545159578323364,
      "logps/chosen": -4.7128987312316895,
      "logps/rejected": -39.89329147338867,
      "loss": 0.7504162192344666,
      "memory(GiB)": 46.71,
      "nll_loss": 0.41977450251579285,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.04398459568619728,
      "rewards/margins": 2.3563807010650635,
      "rewards/rejected": -2.400365114212036,
      "step": 140,
      "train_speed(iter/s)": 0.011182
    },
    {
      "epoch": 0.31464435146443515,
      "grad_norm": 2.3725013732910156,
      "learning_rate": 0.00019378554880734527,
      "logits/chosen": -0.25585880875587463,
      "logits/rejected": -0.40883561968803406,
      "logps/chosen": -5.454152584075928,
      "logps/rejected": -47.51259994506836,
      "loss": 0.6222885251045227,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3938242197036743,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09286467730998993,
      "rewards/margins": 3.4271411895751953,
      "rewards/rejected": -3.3342764377593994,
      "step": 141,
      "train_speed(iter/s)": 0.011183
    },
    {
      "epoch": 0.3168758716875872,
      "grad_norm": 3.706570863723755,
      "learning_rate": 0.00019365680021890506,
      "logits/chosen": -0.26097267866134644,
      "logits/rejected": -0.3484109938144684,
      "logps/chosen": -8.48320198059082,
      "logps/rejected": -31.970016479492188,
      "loss": 1.0012670755386353,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5183718204498291,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.14288514852523804,
      "rewards/margins": 1.521733045578003,
      "rewards/rejected": -1.6646180152893066,
      "step": 142,
      "train_speed(iter/s)": 0.011183
    },
    {
      "epoch": 0.31910739191073917,
      "grad_norm": 4.2724609375,
      "learning_rate": 0.0001935267752520591,
      "logits/chosen": -0.294653058052063,
      "logits/rejected": -0.40346813201904297,
      "logps/chosen": -9.981515884399414,
      "logps/rejected": -44.07274627685547,
      "loss": 0.8689901828765869,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5607314705848694,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2522781491279602,
      "rewards/margins": 2.5947649478912354,
      "rewards/rejected": -2.847043037414551,
      "step": 143,
      "train_speed(iter/s)": 0.011183
    },
    {
      "epoch": 0.3213389121338912,
      "grad_norm": 6.414367198944092,
      "learning_rate": 0.00019339547567882024,
      "logits/chosen": -0.20774677395820618,
      "logits/rejected": -0.3707664906978607,
      "logps/chosen": -9.347281455993652,
      "logps/rejected": -48.28213882446289,
      "loss": 0.9361674189567566,
      "memory(GiB)": 46.71,
      "nll_loss": 0.667937159538269,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2894432842731476,
      "rewards/margins": 3.465684175491333,
      "rewards/rejected": -3.1762406826019287,
      "step": 144,
      "train_speed(iter/s)": 0.011184
    },
    {
      "epoch": 0.32357043235704325,
      "grad_norm": 4.899329662322998,
      "learning_rate": 0.0001932629032885721,
      "logits/chosen": -0.26781731843948364,
      "logits/rejected": -0.36117565631866455,
      "logps/chosen": -12.092925071716309,
      "logps/rejected": -45.867366790771484,
      "loss": 0.9857743382453918,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6580345630645752,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.3960377275943756,
      "rewards/margins": 2.526942729949951,
      "rewards/rejected": -2.9229800701141357,
      "step": 145,
      "train_speed(iter/s)": 0.011184
    },
    {
      "epoch": 0.32580195258019523,
      "grad_norm": 2.351534843444824,
      "learning_rate": 0.0001931290598880445,
      "logits/chosen": -0.21401235461235046,
      "logits/rejected": -0.3749268651008606,
      "logps/chosen": -6.150831699371338,
      "logps/rejected": -38.348854064941406,
      "loss": 0.6613858342170715,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4117865562438965,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14331001043319702,
      "rewards/margins": 2.4033961296081543,
      "rewards/rejected": -2.2600862979888916,
      "step": 146,
      "train_speed(iter/s)": 0.011184
    },
    {
      "epoch": 0.32803347280334727,
      "grad_norm": 3.732591152191162,
      "learning_rate": 0.00019299394730128895,
      "logits/chosen": -0.2112840712070465,
      "logits/rejected": -0.29343438148498535,
      "logps/chosen": -12.490756034851074,
      "logps/rejected": -47.45105743408203,
      "loss": 0.8992500901222229,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5636640787124634,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10838784277439117,
      "rewards/margins": 2.9080140590667725,
      "rewards/rejected": -3.016401767730713,
      "step": 147,
      "train_speed(iter/s)": 0.011184
    },
    {
      "epoch": 0.3302649930264993,
      "grad_norm": 3.371856927871704,
      "learning_rate": 0.00019285756736965367,
      "logits/chosen": -0.2626963257789612,
      "logits/rejected": -0.3471822142601013,
      "logps/chosen": -7.939330101013184,
      "logps/rejected": -30.135757446289062,
      "loss": 0.9493183493614197,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5397716760635376,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.016782023012638092,
      "rewards/margins": 1.6466865539550781,
      "rewards/rejected": -1.6299041509628296,
      "step": 148,
      "train_speed(iter/s)": 0.011184
    },
    {
      "epoch": 0.33249651324965135,
      "grad_norm": 2.4161276817321777,
      "learning_rate": 0.00019271992195175875,
      "logits/chosen": -0.3086288869380951,
      "logits/rejected": -0.41329652070999146,
      "logps/chosen": -8.446435928344727,
      "logps/rejected": -36.33450698852539,
      "loss": 0.9108085632324219,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5454064607620239,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.148118793964386,
      "rewards/margins": 2.209937810897827,
      "rewards/rejected": -2.061818838119507,
      "step": 149,
      "train_speed(iter/s)": 0.011185
    },
    {
      "epoch": 0.33472803347280333,
      "grad_norm": 3.4835684299468994,
      "learning_rate": 0.00019258101292347042,
      "logits/chosen": -0.22059586644172668,
      "logits/rejected": -0.3747985363006592,
      "logps/chosen": -8.812771797180176,
      "logps/rejected": -44.0800895690918,
      "loss": 0.7637215852737427,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5271515250205994,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.041512295603752136,
      "rewards/margins": 2.4112486839294434,
      "rewards/rejected": -2.369736433029175,
      "step": 150,
      "train_speed(iter/s)": 0.011185
    },
    {
      "epoch": 0.33695955369595537,
      "grad_norm": 2.394838333129883,
      "learning_rate": 0.00019244084217787588,
      "logits/chosen": -0.24826237559318542,
      "logits/rejected": -0.3791409730911255,
      "logps/chosen": -8.393878936767578,
      "logps/rejected": -45.125953674316406,
      "loss": 0.7294331192970276,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4361075460910797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18614013493061066,
      "rewards/margins": 2.6040918827056885,
      "rewards/rejected": -2.417951822280884,
      "step": 151,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.3391910739191074,
      "grad_norm": 2.946138620376587,
      "learning_rate": 0.00019229941162525726,
      "logits/chosen": -0.2787875831127167,
      "logits/rejected": -0.3120588958263397,
      "logps/chosen": -9.058300018310547,
      "logps/rejected": -30.770740509033203,
      "loss": 0.9096699357032776,
      "memory(GiB)": 46.71,
      "nll_loss": 0.46970880031585693,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.08315091580152512,
      "rewards/margins": 1.5154523849487305,
      "rewards/rejected": -1.5986034870147705,
      "step": 152,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.3414225941422594,
      "grad_norm": 2.37446928024292,
      "learning_rate": 0.00019215672319306565,
      "logits/chosen": -0.29989758133888245,
      "logits/rejected": -0.372408926486969,
      "logps/chosen": -6.495873928070068,
      "logps/rejected": -32.42622375488281,
      "loss": 0.7089872360229492,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3819705545902252,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0852634608745575,
      "rewards/margins": 1.8665060997009277,
      "rewards/rejected": -1.7812427282333374,
      "step": 153,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.34365411436541143,
      "grad_norm": 1.6320723295211792,
      "learning_rate": 0.00019201277882589487,
      "logits/chosen": -0.2685769498348236,
      "logits/rejected": -0.4338504374027252,
      "logps/chosen": -4.207334995269775,
      "logps/rejected": -54.69655990600586,
      "loss": 0.4186314344406128,
      "memory(GiB)": 46.71,
      "nll_loss": 0.23857831954956055,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.104047991335392,
      "rewards/margins": 3.6357991695404053,
      "rewards/rejected": -3.5317511558532715,
      "step": 154,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.3458856345885635,
      "grad_norm": 2.5706071853637695,
      "learning_rate": 0.00019186758048545498,
      "logits/chosen": -0.31499671936035156,
      "logits/rejected": -0.4452478885650635,
      "logps/chosen": -8.811934471130371,
      "logps/rejected": -41.911869049072266,
      "loss": 0.8359880447387695,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5718236565589905,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.00384414941072464,
      "rewards/margins": 2.2188119888305664,
      "rewards/rejected": -2.214968204498291,
      "step": 155,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.34811715481171546,
      "grad_norm": 3.741011619567871,
      "learning_rate": 0.00019172113015054532,
      "logits/chosen": -0.42364075779914856,
      "logits/rejected": -0.48594316840171814,
      "logps/chosen": -9.2898588180542,
      "logps/rejected": -39.12040710449219,
      "loss": 0.7001189589500427,
      "memory(GiB)": 46.71,
      "nll_loss": 0.43081939220428467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08180733025074005,
      "rewards/margins": 2.411517381668091,
      "rewards/rejected": -2.329710006713867,
      "step": 156,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.3503486750348675,
      "grad_norm": 2.2846014499664307,
      "learning_rate": 0.00019157342981702792,
      "logits/chosen": -0.34239232540130615,
      "logits/rejected": -0.5085087418556213,
      "logps/chosen": -3.7471375465393066,
      "logps/rejected": -39.35340118408203,
      "loss": 0.537229597568512,
      "memory(GiB)": 46.71,
      "nll_loss": 0.32565146684646606,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18335413932800293,
      "rewards/margins": 2.308189868927002,
      "rewards/rejected": -2.124835968017578,
      "step": 157,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.35258019525801954,
      "grad_norm": 2.8782355785369873,
      "learning_rate": 0.0001914244814978001,
      "logits/chosen": -0.36600109934806824,
      "logits/rejected": -0.4807679057121277,
      "logps/chosen": -12.43294906616211,
      "logps/rejected": -36.17433547973633,
      "loss": 0.7901278734207153,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5272985100746155,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.03357819840312004,
      "rewards/margins": 2.113076686859131,
      "rewards/rejected": -2.079498291015625,
      "step": 158,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.3548117154811716,
      "grad_norm": 4.173655033111572,
      "learning_rate": 0.00019127428722276685,
      "logits/chosen": -0.29084256291389465,
      "logits/rejected": -0.49927467107772827,
      "logps/chosen": -8.242868423461914,
      "logps/rejected": -54.8849983215332,
      "loss": 0.7512383460998535,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5454739928245544,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08500596880912781,
      "rewards/margins": 3.6066102981567383,
      "rewards/rejected": -3.521604299545288,
      "step": 159,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.35704323570432356,
      "grad_norm": 2.355539083480835,
      "learning_rate": 0.0001911228490388136,
      "logits/chosen": -0.3650870621204376,
      "logits/rejected": -0.5237394571304321,
      "logps/chosen": -10.184951782226562,
      "logps/rejected": -48.1030387878418,
      "loss": 0.7325472831726074,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5253479480743408,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15602552890777588,
      "rewards/margins": 3.2045021057128906,
      "rewards/rejected": -3.048476457595825,
      "step": 160,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.3592747559274756,
      "grad_norm": 4.184498310089111,
      "learning_rate": 0.00019097016900977788,
      "logits/chosen": -0.41671380400657654,
      "logits/rejected": -0.5081248879432678,
      "logps/chosen": -10.973024368286133,
      "logps/rejected": -47.46335983276367,
      "loss": 0.5701618194580078,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3158893883228302,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09667229652404785,
      "rewards/margins": 3.3210151195526123,
      "rewards/rejected": -3.2243423461914062,
      "step": 161,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.36150627615062764,
      "grad_norm": 4.150604724884033,
      "learning_rate": 0.00019081624921642157,
      "logits/chosen": -0.36136606335639954,
      "logits/rejected": -0.5436823964118958,
      "logps/chosen": -7.6842756271362305,
      "logps/rejected": -45.09693908691406,
      "loss": 0.7053879499435425,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4340396523475647,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07871361076831818,
      "rewards/margins": 2.7361950874328613,
      "rewards/rejected": -2.6574814319610596,
      "step": 162,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.3637377963737796,
      "grad_norm": 3.704453706741333,
      "learning_rate": 0.00019066109175640224,
      "logits/chosen": -0.40510815382003784,
      "logits/rejected": -0.4312695860862732,
      "logps/chosen": -12.371721267700195,
      "logps/rejected": -50.284141540527344,
      "loss": 0.7488542795181274,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4703201949596405,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1792062222957611,
      "rewards/margins": 3.446262836456299,
      "rewards/rejected": -3.267056941986084,
      "step": 163,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.36596931659693166,
      "grad_norm": 2.7603952884674072,
      "learning_rate": 0.00019050469874424476,
      "logits/chosen": -0.32453101873397827,
      "logits/rejected": -0.4822920560836792,
      "logps/chosen": -8.248761177062988,
      "logps/rejected": -45.44584655761719,
      "loss": 0.8071178197860718,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4910924434661865,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.050887446850538254,
      "rewards/margins": 3.054812431335449,
      "rewards/rejected": -3.00392484664917,
      "step": 164,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.3682008368200837,
      "grad_norm": 5.73638916015625,
      "learning_rate": 0.0001903470723113124,
      "logits/chosen": -0.37211766839027405,
      "logits/rejected": -0.36960119009017944,
      "logps/chosen": -13.764412879943848,
      "logps/rejected": -28.488418579101562,
      "loss": 1.0636341571807861,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5649504661560059,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.17467395961284637,
      "rewards/margins": 1.285156488418579,
      "rewards/rejected": -1.459830403327942,
      "step": 165,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.3704323570432357,
      "grad_norm": 3.937652587890625,
      "learning_rate": 0.00019018821460577777,
      "logits/chosen": -0.2960231602191925,
      "logits/rejected": -0.4136447012424469,
      "logps/chosen": -12.963543891906738,
      "logps/rejected": -43.19397735595703,
      "loss": 0.8952102065086365,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5876767039299011,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.3215300142765045,
      "rewards/margins": 2.2675259113311768,
      "rewards/rejected": -2.5890560150146484,
      "step": 166,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.3726638772663877,
      "grad_norm": 2.5968947410583496,
      "learning_rate": 0.00019002812779259364,
      "logits/chosen": -0.31316545605659485,
      "logits/rejected": -0.510556161403656,
      "logps/chosen": -6.747772216796875,
      "logps/rejected": -37.75751876831055,
      "loss": 0.7592855095863342,
      "memory(GiB)": 46.71,
      "nll_loss": 0.47158926725387573,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.17958350479602814,
      "rewards/margins": 2.317811965942383,
      "rewards/rejected": -2.138228416442871,
      "step": 167,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.37489539748953976,
      "grad_norm": 9.810136795043945,
      "learning_rate": 0.00018986681405346322,
      "logits/chosen": -0.28982365131378174,
      "logits/rejected": -0.48794350028038025,
      "logps/chosen": -9.523417472839355,
      "logps/rejected": -53.576263427734375,
      "loss": 0.8609976172447205,
      "memory(GiB)": 46.71,
      "nll_loss": 0.55437833070755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.018897701054811478,
      "rewards/margins": 3.141695499420166,
      "rewards/rejected": -3.160593032836914,
      "step": 168,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.37712691771269174,
      "grad_norm": 3.2620904445648193,
      "learning_rate": 0.00018970427558681082,
      "logits/chosen": -0.2701128125190735,
      "logits/rejected": -0.5124972462654114,
      "logps/chosen": -7.385637283325195,
      "logps/rejected": -66.33370971679688,
      "loss": 0.595870852470398,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4602699279785156,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2253667265176773,
      "rewards/margins": 4.247678756713867,
      "rewards/rejected": -4.022312164306641,
      "step": 169,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.3793584379358438,
      "grad_norm": 2.7175841331481934,
      "learning_rate": 0.0001895405146077514,
      "logits/chosen": -0.3605695366859436,
      "logits/rejected": -0.521330714225769,
      "logps/chosen": -5.586010456085205,
      "logps/rejected": -52.13179397583008,
      "loss": 0.652187705039978,
      "memory(GiB)": 46.71,
      "nll_loss": 0.32753562927246094,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04342030733823776,
      "rewards/margins": 3.0307912826538086,
      "rewards/rejected": -3.074211597442627,
      "step": 170,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.3815899581589958,
      "grad_norm": 3.162261486053467,
      "learning_rate": 0.00018937553334806077,
      "logits/chosen": -0.29991668462753296,
      "logits/rejected": -0.4432262182235718,
      "logps/chosen": -9.342361450195312,
      "logps/rejected": -48.89967346191406,
      "loss": 0.7870311737060547,
      "memory(GiB)": 46.71,
      "nll_loss": 0.543519139289856,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10350081324577332,
      "rewards/margins": 3.252532720565796,
      "rewards/rejected": -3.149031639099121,
      "step": 171,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.38382147838214786,
      "grad_norm": 11.82007122039795,
      "learning_rate": 0.000189209334056145,
      "logits/chosen": -0.3029250502586365,
      "logits/rejected": -0.4914131164550781,
      "logps/chosen": -5.792529582977295,
      "logps/rejected": -52.44511413574219,
      "loss": 0.6816909313201904,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5198902487754822,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.058606795966625214,
      "rewards/margins": 3.3576695919036865,
      "rewards/rejected": -3.299063205718994,
      "step": 172,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.38605299860529985,
      "grad_norm": 2.9290287494659424,
      "learning_rate": 0.00018904191899700977,
      "logits/chosen": -0.40706759691238403,
      "logits/rejected": -0.45858079195022583,
      "logps/chosen": -6.328141212463379,
      "logps/rejected": -34.812583923339844,
      "loss": 0.7009620666503906,
      "memory(GiB)": 46.71,
      "nll_loss": 0.44378477334976196,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12429780513048172,
      "rewards/margins": 2.033411979675293,
      "rewards/rejected": -1.909114122390747,
      "step": 173,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.3882845188284519,
      "grad_norm": 3.7593777179718018,
      "learning_rate": 0.0001888732904522296,
      "logits/chosen": -0.380102276802063,
      "logits/rejected": -0.4688776731491089,
      "logps/chosen": -5.4205002784729,
      "logps/rejected": -42.66455078125,
      "loss": 0.705348014831543,
      "memory(GiB)": 46.71,
      "nll_loss": 0.48004987835884094,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09237709641456604,
      "rewards/margins": 2.7925949096679688,
      "rewards/rejected": -2.7002179622650146,
      "step": 174,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.3905160390516039,
      "grad_norm": 4.404231548309326,
      "learning_rate": 0.00018870345071991663,
      "logits/chosen": -0.42475855350494385,
      "logits/rejected": -0.49599534273147583,
      "logps/chosen": -4.999125957489014,
      "logps/rejected": -31.932191848754883,
      "loss": 0.5820786356925964,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3192511200904846,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.01989058405160904,
      "rewards/margins": 1.8925269842147827,
      "rewards/rejected": -1.8726365566253662,
      "step": 175,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.3927475592747559,
      "grad_norm": 4.2940239906311035,
      "learning_rate": 0.00018853240211468944,
      "logits/chosen": -0.393295019865036,
      "logits/rejected": -0.4561651647090912,
      "logps/chosen": -12.356813430786133,
      "logps/rejected": -33.95934295654297,
      "loss": 0.9481363892555237,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5598008036613464,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.053550563752651215,
      "rewards/margins": 1.7341490983963013,
      "rewards/rejected": -1.680598497390747,
      "step": 176,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.39497907949790795,
      "grad_norm": 5.875354766845703,
      "learning_rate": 0.0001883601469676414,
      "logits/chosen": -0.36429429054260254,
      "logits/rejected": -0.5053366422653198,
      "logps/chosen": -4.228557586669922,
      "logps/rejected": -41.48766326904297,
      "loss": 0.5440742373466492,
      "memory(GiB)": 46.71,
      "nll_loss": 0.34625735878944397,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.046092353761196136,
      "rewards/margins": 2.599119186401367,
      "rewards/rejected": -2.5530271530151367,
      "step": 177,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.39721059972106,
      "grad_norm": 2.555330753326416,
      "learning_rate": 0.00018818668762630888,
      "logits/chosen": -0.40784329175949097,
      "logits/rejected": -0.5250139236450195,
      "logps/chosen": -6.325516700744629,
      "logps/rejected": -37.756141662597656,
      "loss": 0.5609665513038635,
      "memory(GiB)": 46.71,
      "nll_loss": 0.36217477917671204,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06898089498281479,
      "rewards/margins": 2.456758499145508,
      "rewards/rejected": -2.38777756690979,
      "step": 178,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.39944211994421197,
      "grad_norm": 2.5570666790008545,
      "learning_rate": 0.00018801202645463947,
      "logits/chosen": -0.49009108543395996,
      "logits/rejected": -0.5421456694602966,
      "logps/chosen": -7.800501823425293,
      "logps/rejected": -31.90920639038086,
      "loss": 0.7761667966842651,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5564302206039429,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22300021350383759,
      "rewards/margins": 2.3164584636688232,
      "rewards/rejected": -2.093458414077759,
      "step": 179,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.401673640167364,
      "grad_norm": 3.062514066696167,
      "learning_rate": 0.00018783616583295943,
      "logits/chosen": -0.3902238607406616,
      "logits/rejected": -0.5166201591491699,
      "logps/chosen": -5.271551132202148,
      "logps/rejected": -43.41081237792969,
      "loss": 0.586236834526062,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3715082108974457,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10219533741474152,
      "rewards/margins": 2.8101999759674072,
      "rewards/rejected": -2.7080047130584717,
      "step": 180,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.40390516039051605,
      "grad_norm": 2.8764405250549316,
      "learning_rate": 0.0001876591081579415,
      "logits/chosen": -0.4361119270324707,
      "logits/rejected": -0.5355557203292847,
      "logps/chosen": -5.954665184020996,
      "logps/rejected": -40.49996566772461,
      "loss": 0.6423499584197998,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3932062089443207,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11792459338903427,
      "rewards/margins": 2.8236496448516846,
      "rewards/rejected": -2.7057251930236816,
      "step": 181,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.4061366806136681,
      "grad_norm": 3.4861021041870117,
      "learning_rate": 0.0001874808558425722,
      "logits/chosen": -0.44822514057159424,
      "logits/rejected": -0.5208377838134766,
      "logps/chosen": -8.507841110229492,
      "logps/rejected": -38.145362854003906,
      "loss": 0.7365833520889282,
      "memory(GiB)": 46.71,
      "nll_loss": 0.45295223593711853,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.18383249640464783,
      "rewards/margins": 2.3255980014801025,
      "rewards/rejected": -2.141765594482422,
      "step": 182,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.40836820083682007,
      "grad_norm": 3.0177836418151855,
      "learning_rate": 0.00018730141131611882,
      "logits/chosen": -0.4113612771034241,
      "logits/rejected": -0.5111112594604492,
      "logps/chosen": -7.983984470367432,
      "logps/rejected": -39.916988372802734,
      "loss": 0.7822529077529907,
      "memory(GiB)": 46.71,
      "nll_loss": 0.48471909761428833,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.010233555920422077,
      "rewards/margins": 2.3506059646606445,
      "rewards/rejected": -2.340372323989868,
      "step": 183,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.4105997210599721,
      "grad_norm": 2.848646879196167,
      "learning_rate": 0.0001871207770240965,
      "logits/chosen": -0.41031956672668457,
      "logits/rejected": -0.5503991842269897,
      "logps/chosen": -6.160996437072754,
      "logps/rejected": -49.401180267333984,
      "loss": 0.7412950992584229,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5057318806648254,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11709622293710709,
      "rewards/margins": 2.803727388381958,
      "rewards/rejected": -2.686631441116333,
      "step": 184,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.41283124128312415,
      "grad_norm": 3.7771029472351074,
      "learning_rate": 0.00018693895542823476,
      "logits/chosen": -0.4378977417945862,
      "logits/rejected": -0.5446797609329224,
      "logps/chosen": -3.902487277984619,
      "logps/rejected": -38.302616119384766,
      "loss": 0.6036807298660278,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3306537866592407,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07923398911952972,
      "rewards/margins": 2.431417465209961,
      "rewards/rejected": -2.3521835803985596,
      "step": 185,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.41506276150627613,
      "grad_norm": 2.6410114765167236,
      "learning_rate": 0.00018675594900644394,
      "logits/chosen": -0.4403599798679352,
      "logits/rejected": -0.534117579460144,
      "logps/chosen": -7.202261447906494,
      "logps/rejected": -40.587371826171875,
      "loss": 0.6803833842277527,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4558661878108978,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11624391376972198,
      "rewards/margins": 2.7324059009552,
      "rewards/rejected": -2.616162061691284,
      "step": 186,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.4172942817294282,
      "grad_norm": 27.184694290161133,
      "learning_rate": 0.0001865717602527816,
      "logits/chosen": -0.3826313018798828,
      "logits/rejected": -0.5364388227462769,
      "logps/chosen": -5.527843952178955,
      "logps/rejected": -49.32733917236328,
      "loss": 0.7175433039665222,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5420017242431641,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13603699207305908,
      "rewards/margins": 3.4389665126800537,
      "rewards/rejected": -3.302929401397705,
      "step": 187,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.4195258019525802,
      "grad_norm": 2.5546696186065674,
      "learning_rate": 0.00018638639167741834,
      "logits/chosen": -0.432650625705719,
      "logits/rejected": -0.5771442651748657,
      "logps/chosen": -6.985705375671387,
      "logps/rejected": -55.28477478027344,
      "loss": 0.5891926884651184,
      "memory(GiB)": 46.71,
      "nll_loss": 0.41381514072418213,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2026616781949997,
      "rewards/margins": 4.167919635772705,
      "rewards/rejected": -3.9652576446533203,
      "step": 188,
      "train_speed(iter/s)": 0.011178
    },
    {
      "epoch": 0.4217573221757322,
      "grad_norm": 2.758708953857422,
      "learning_rate": 0.00018619984580660367,
      "logits/chosen": -0.41937315464019775,
      "logits/rejected": -0.5321905612945557,
      "logps/chosen": -6.989339828491211,
      "logps/rejected": -58.12518310546875,
      "loss": 0.5879656672477722,
      "memory(GiB)": 46.71,
      "nll_loss": 0.36345142126083374,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.023294318467378616,
      "rewards/margins": 4.040982246398926,
      "rewards/rejected": -4.064276695251465,
      "step": 189,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.42398884239888424,
      "grad_norm": 11.678010940551758,
      "learning_rate": 0.00018601212518263156,
      "logits/chosen": -0.39216288924217224,
      "logits/rejected": -0.4594748020172119,
      "logps/chosen": -9.29268741607666,
      "logps/rejected": -54.65095520019531,
      "loss": 0.818715512752533,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6221151947975159,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13676013052463531,
      "rewards/margins": 4.018932819366455,
      "rewards/rejected": -3.8821723461151123,
      "step": 190,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.4262203626220363,
      "grad_norm": 3.9322588443756104,
      "learning_rate": 0.00018582323236380588,
      "logits/chosen": -0.4796799421310425,
      "logits/rejected": -0.5020566582679749,
      "logps/chosen": -14.528449058532715,
      "logps/rejected": -43.279300689697266,
      "loss": 0.9906633496284485,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6794536709785461,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2910754680633545,
      "rewards/margins": 2.7544803619384766,
      "rewards/rejected": -3.045555591583252,
      "step": 191,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.42845188284518826,
      "grad_norm": 5.284152507781982,
      "learning_rate": 0.00018563316992440543,
      "logits/chosen": -0.4428099989891052,
      "logits/rejected": -0.5434530973434448,
      "logps/chosen": -6.242356777191162,
      "logps/rejected": -46.20015335083008,
      "loss": 0.8040673732757568,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4949415922164917,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05515892803668976,
      "rewards/margins": 2.9070026874542236,
      "rewards/rejected": -2.9621615409851074,
      "step": 192,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.4306834030683403,
      "grad_norm": 11.493171691894531,
      "learning_rate": 0.00018544194045464886,
      "logits/chosen": -0.38638293743133545,
      "logits/rejected": -0.4842715561389923,
      "logps/chosen": -6.438551425933838,
      "logps/rejected": -44.880149841308594,
      "loss": 0.7885379791259766,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5359107255935669,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.19068023562431335,
      "rewards/margins": 2.8829379081726074,
      "rewards/rejected": -2.692257881164551,
      "step": 193,
      "train_speed(iter/s)": 0.011179
    },
    {
      "epoch": 0.43291492329149234,
      "grad_norm": 2.3137385845184326,
      "learning_rate": 0.00018524954656065946,
      "logits/chosen": -0.4063287675380707,
      "logits/rejected": -0.5103312730789185,
      "logps/chosen": -7.378753662109375,
      "logps/rejected": -45.85976791381836,
      "loss": 0.45938000082969666,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3061022460460663,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08831208944320679,
      "rewards/margins": 3.123901605606079,
      "rewards/rejected": -3.0355896949768066,
      "step": 194,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.4351464435146444,
      "grad_norm": 4.795388221740723,
      "learning_rate": 0.00018505599086442956,
      "logits/chosen": -0.38035646080970764,
      "logits/rejected": -0.5444158315658569,
      "logps/chosen": -2.437512159347534,
      "logps/rejected": -43.048858642578125,
      "loss": 0.5650367736816406,
      "memory(GiB)": 46.71,
      "nll_loss": 0.33980026841163635,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13193824887275696,
      "rewards/margins": 2.7752439975738525,
      "rewards/rejected": -2.643305540084839,
      "step": 195,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.43737796373779636,
      "grad_norm": 2.4795217514038086,
      "learning_rate": 0.0001848612760037848,
      "logits/chosen": -0.3798023462295532,
      "logits/rejected": -0.5086287260055542,
      "logps/chosen": -4.153357982635498,
      "logps/rejected": -47.3394775390625,
      "loss": 0.4092462360858917,
      "memory(GiB)": 46.71,
      "nll_loss": 0.22234448790550232,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1915455460548401,
      "rewards/margins": 2.8723514080047607,
      "rewards/rejected": -2.6808059215545654,
      "step": 196,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.4396094839609484,
      "grad_norm": 4.820924282073975,
      "learning_rate": 0.0001846654046323482,
      "logits/chosen": -0.41246578097343445,
      "logits/rejected": -0.4796575903892517,
      "logps/chosen": -10.3787841796875,
      "logps/rejected": -37.14994812011719,
      "loss": 0.9857741594314575,
      "memory(GiB)": 46.71,
      "nll_loss": 0.669836163520813,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.05453118681907654,
      "rewards/margins": 2.0403127670288086,
      "rewards/rejected": -2.094843864440918,
      "step": 197,
      "train_speed(iter/s)": 0.01118
    },
    {
      "epoch": 0.44184100418410044,
      "grad_norm": 3.00054931640625,
      "learning_rate": 0.0001844683794195041,
      "logits/chosen": -0.3768506646156311,
      "logits/rejected": -0.5438990592956543,
      "logps/chosen": -6.3500471115112305,
      "logps/rejected": -49.48595428466797,
      "loss": 0.6618455648422241,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4003591537475586,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.02325659617781639,
      "rewards/margins": 3.2074074745178223,
      "rewards/rejected": -3.1841506958007812,
      "step": 198,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.4440725244072524,
      "grad_norm": 5.5015339851379395,
      "learning_rate": 0.00018427020305036157,
      "logits/chosen": -0.43915611505508423,
      "logits/rejected": -0.5123714804649353,
      "logps/chosen": -5.833986759185791,
      "logps/rejected": -41.57982635498047,
      "loss": 0.7684890031814575,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4649631679058075,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04928220808506012,
      "rewards/margins": 2.619971752166748,
      "rewards/rejected": -2.6692538261413574,
      "step": 199,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.44630404463040446,
      "grad_norm": 4.47312068939209,
      "learning_rate": 0.00018407087822571794,
      "logits/chosen": -0.41636520624160767,
      "logits/rejected": -0.5203005075454712,
      "logps/chosen": -9.286731719970703,
      "logps/rejected": -41.12049102783203,
      "loss": 0.8878820538520813,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6213392615318298,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1641845703125,
      "rewards/margins": 2.593937397003174,
      "rewards/rejected": -2.4297525882720947,
      "step": 200,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.4485355648535565,
      "grad_norm": 2.523305654525757,
      "learning_rate": 0.00018387040766202202,
      "logits/chosen": -0.43077439069747925,
      "logits/rejected": -0.5606883764266968,
      "logps/chosen": -9.173616409301758,
      "logps/rejected": -54.819252014160156,
      "loss": 0.6095305681228638,
      "memory(GiB)": 46.71,
      "nll_loss": 0.43501588702201843,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17188499867916107,
      "rewards/margins": 3.581376791000366,
      "rewards/rejected": -3.409491539001465,
      "step": 201,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.4507670850767085,
      "grad_norm": 10.484657287597656,
      "learning_rate": 0.00018366879409133702,
      "logits/chosen": -0.46561458706855774,
      "logits/rejected": -0.5806446075439453,
      "logps/chosen": -9.230829238891602,
      "logps/rejected": -50.296417236328125,
      "loss": 0.8357149362564087,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5318824052810669,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04186338931322098,
      "rewards/margins": 3.0313618183135986,
      "rewards/rejected": -3.0732247829437256,
      "step": 202,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.4529986052998605,
      "grad_norm": 3.0593442916870117,
      "learning_rate": 0.00018346604026130335,
      "logits/chosen": -0.4838520884513855,
      "logits/rejected": -0.563227117061615,
      "logps/chosen": -6.622610092163086,
      "logps/rejected": -33.531341552734375,
      "loss": 0.7672325372695923,
      "memory(GiB)": 46.71,
      "nll_loss": 0.47228333353996277,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16688543558120728,
      "rewards/margins": 2.1572673320770264,
      "rewards/rejected": -1.9903819561004639,
      "step": 203,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.45523012552301256,
      "grad_norm": 3.2974071502685547,
      "learning_rate": 0.00018326214893510113,
      "logits/chosen": -0.48938241600990295,
      "logits/rejected": -0.5406135320663452,
      "logps/chosen": -6.855633735656738,
      "logps/rejected": -32.516056060791016,
      "loss": 0.7795042991638184,
      "memory(GiB)": 46.71,
      "nll_loss": 0.463142991065979,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.058080192655324936,
      "rewards/margins": 2.187054395675659,
      "rewards/rejected": -2.128974199295044,
      "step": 204,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.45746164574616455,
      "grad_norm": 2.7023813724517822,
      "learning_rate": 0.00018305712289141263,
      "logits/chosen": -0.503454864025116,
      "logits/rejected": -0.5921227931976318,
      "logps/chosen": -7.617156982421875,
      "logps/rejected": -42.7328987121582,
      "loss": 0.6725366711616516,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3977656364440918,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02913985215127468,
      "rewards/margins": 2.757948398590088,
      "rewards/rejected": -2.728808641433716,
      "step": 205,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.4596931659693166,
      "grad_norm": 3.1167116165161133,
      "learning_rate": 0.00018285096492438424,
      "logits/chosen": -0.5165062546730042,
      "logits/rejected": -0.6217053532600403,
      "logps/chosen": -7.3234543800354,
      "logps/rejected": -50.999603271484375,
      "loss": 0.6318544745445251,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3780009150505066,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.00554865226149559,
      "rewards/margins": 3.287566661834717,
      "rewards/rejected": -3.2820186614990234,
      "step": 206,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.4619246861924686,
      "grad_norm": 9.524746894836426,
      "learning_rate": 0.00018264367784358854,
      "logits/chosen": -0.4920929968357086,
      "logits/rejected": -0.5764138102531433,
      "logps/chosen": -7.860991954803467,
      "logps/rejected": -42.47827911376953,
      "loss": 0.7360129356384277,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5072849988937378,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2539522051811218,
      "rewards/margins": 2.9447669982910156,
      "rewards/rejected": -2.690814733505249,
      "step": 207,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.46415620641562066,
      "grad_norm": 3.1537561416625977,
      "learning_rate": 0.00018243526447398595,
      "logits/chosen": -0.49637073278427124,
      "logits/rejected": -0.5572264194488525,
      "logps/chosen": -7.864004135131836,
      "logps/rejected": -36.40027618408203,
      "loss": 0.701461911201477,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4231345057487488,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05218251794576645,
      "rewards/margins": 2.490140438079834,
      "rewards/rejected": -2.437958002090454,
      "step": 208,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.46638772663877265,
      "grad_norm": 5.860493183135986,
      "learning_rate": 0.00018222572765588625,
      "logits/chosen": -0.4768957793712616,
      "logits/rejected": -0.5333793759346008,
      "logps/chosen": -14.414131164550781,
      "logps/rejected": -43.239593505859375,
      "loss": 0.8101565837860107,
      "memory(GiB)": 46.71,
      "nll_loss": 0.49351921677589417,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05914236605167389,
      "rewards/margins": 2.7074344158172607,
      "rewards/rejected": -2.648291826248169,
      "step": 209,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.4686192468619247,
      "grad_norm": 2.79776930809021,
      "learning_rate": 0.00018201507024490988,
      "logits/chosen": -0.4704314172267914,
      "logits/rejected": -0.567833662033081,
      "logps/chosen": -14.108932495117188,
      "logps/rejected": -44.836395263671875,
      "loss": 0.8302231431007385,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6059545278549194,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17180928587913513,
      "rewards/margins": 2.3872344493865967,
      "rewards/rejected": -2.2154252529144287,
      "step": 210,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.4708507670850767,
      "grad_norm": 1.4490225315093994,
      "learning_rate": 0.0001818032951119489,
      "logits/chosen": -0.45868298411369324,
      "logits/rejected": -0.6028948426246643,
      "logps/chosen": -3.0846667289733887,
      "logps/rejected": -51.68486404418945,
      "loss": 0.36262989044189453,
      "memory(GiB)": 46.71,
      "nll_loss": 0.25144004821777344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2839798927307129,
      "rewards/margins": 3.57785964012146,
      "rewards/rejected": -3.293879747390747,
      "step": 211,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.4730822873082287,
      "grad_norm": 2.7278997898101807,
      "learning_rate": 0.00018159040514312803,
      "logits/chosen": -0.5122844576835632,
      "logits/rejected": -0.5693905353546143,
      "logps/chosen": -8.715449333190918,
      "logps/rejected": -31.896915435791016,
      "loss": 0.7132190465927124,
      "memory(GiB)": 46.71,
      "nll_loss": 0.39171797037124634,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2175295054912567,
      "rewards/margins": 2.153326988220215,
      "rewards/rejected": -1.9357972145080566,
      "step": 212,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.47531380753138075,
      "grad_norm": 2.76104474067688,
      "learning_rate": 0.00018137640323976536,
      "logits/chosen": -0.5042344331741333,
      "logits/rejected": -0.5739261507987976,
      "logps/chosen": -12.65793228149414,
      "logps/rejected": -42.124107360839844,
      "loss": 0.7560016512870789,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5502416491508484,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.03646032512187958,
      "rewards/margins": 2.767151355743408,
      "rewards/rejected": -2.7306909561157227,
      "step": 213,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.4775453277545328,
      "grad_norm": 4.021885871887207,
      "learning_rate": 0.00018116129231833249,
      "logits/chosen": -0.466595321893692,
      "logits/rejected": -0.5636504888534546,
      "logps/chosen": -8.149341583251953,
      "logps/rejected": -38.28250503540039,
      "loss": 0.7789570093154907,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4304181933403015,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0456395149230957,
      "rewards/margins": 2.337042808532715,
      "rewards/rejected": -2.2914035320281982,
      "step": 214,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.4797768479776848,
      "grad_norm": 2.7031266689300537,
      "learning_rate": 0.00018094507531041517,
      "logits/chosen": -0.5194039940834045,
      "logits/rejected": -0.5763382315635681,
      "logps/chosen": -12.234911918640137,
      "logps/rejected": -41.63436508178711,
      "loss": 0.6696392893791199,
      "memory(GiB)": 46.71,
      "nll_loss": 0.397487074136734,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.020977359265089035,
      "rewards/margins": 2.5402636528015137,
      "rewards/rejected": -2.5192863941192627,
      "step": 215,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.4820083682008368,
      "grad_norm": 4.239037990570068,
      "learning_rate": 0.00018072775516267306,
      "logits/chosen": -0.4820006191730499,
      "logits/rejected": -0.5642452239990234,
      "logps/chosen": -5.411629676818848,
      "logps/rejected": -49.11735534667969,
      "loss": 0.6698524951934814,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4091280698776245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.007008063141256571,
      "rewards/margins": 2.734046459197998,
      "rewards/rejected": -2.7270381450653076,
      "step": 216,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.48423988842398885,
      "grad_norm": 2.3891801834106445,
      "learning_rate": 0.00018050933483679976,
      "logits/chosen": -0.48481085896492004,
      "logits/rejected": -0.6319360136985779,
      "logps/chosen": -6.989506721496582,
      "logps/rejected": -56.728424072265625,
      "loss": 0.601786732673645,
      "memory(GiB)": 46.71,
      "nll_loss": 0.35297003388404846,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.13874617218971252,
      "rewards/margins": 3.526907205581665,
      "rewards/rejected": -3.3881607055664062,
      "step": 217,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.4864714086471409,
      "grad_norm": 6.219696044921875,
      "learning_rate": 0.0001802898173094824,
      "logits/chosen": -0.5356330871582031,
      "logits/rejected": -0.6473451852798462,
      "logps/chosen": -8.472892761230469,
      "logps/rejected": -51.16962432861328,
      "loss": 0.783514142036438,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5371319651603699,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06705744564533234,
      "rewards/margins": 3.750828266143799,
      "rewards/rejected": -3.6837711334228516,
      "step": 218,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.4887029288702929,
      "grad_norm": 6.937330722808838,
      "learning_rate": 0.0001800692055723609,
      "logits/chosen": -0.6238746047019958,
      "logits/rejected": -0.6939642429351807,
      "logps/chosen": -4.523331642150879,
      "logps/rejected": -37.64277267456055,
      "loss": 0.6180140376091003,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3569566607475281,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11646758764982224,
      "rewards/margins": 2.8116490840911865,
      "rewards/rejected": -2.695181369781494,
      "step": 219,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.4909344490934449,
      "grad_norm": 4.587141036987305,
      "learning_rate": 0.0001798475026319875,
      "logits/chosen": -0.5524365901947021,
      "logits/rejected": -0.7282678484916687,
      "logps/chosen": -4.9026618003845215,
      "logps/rejected": -58.83467483520508,
      "loss": 0.5599265098571777,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3686913251876831,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08274698257446289,
      "rewards/margins": 4.399111747741699,
      "rewards/rejected": -4.316364765167236,
      "step": 220,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.49316596931659695,
      "grad_norm": 19.358779907226562,
      "learning_rate": 0.00017962471150978563,
      "logits/chosen": -0.6010394096374512,
      "logits/rejected": -0.7070757150650024,
      "logps/chosen": -7.744332313537598,
      "logps/rejected": -56.298973083496094,
      "loss": 1.0826630592346191,
      "memory(GiB)": 46.71,
      "nll_loss": 0.7130725979804993,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.036551520228385925,
      "rewards/margins": 4.22437858581543,
      "rewards/rejected": -4.260930061340332,
      "step": 221,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.49539748953974894,
      "grad_norm": 1.9726710319519043,
      "learning_rate": 0.00017940083524200858,
      "logits/chosen": -0.5176704525947571,
      "logits/rejected": -0.6265690326690674,
      "logps/chosen": -8.897793769836426,
      "logps/rejected": -51.35803985595703,
      "loss": 0.5861097574234009,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4163362979888916,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3185967803001404,
      "rewards/margins": 3.5779831409454346,
      "rewards/rejected": -3.2593865394592285,
      "step": 222,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.497629009762901,
      "grad_norm": 2.865757465362549,
      "learning_rate": 0.00017917587687969847,
      "logits/chosen": -0.5248193740844727,
      "logits/rejected": -0.6183274984359741,
      "logps/chosen": -11.439458847045898,
      "logps/rejected": -43.991424560546875,
      "loss": 0.8551426529884338,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6576440334320068,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24198690056800842,
      "rewards/margins": 3.191100597381592,
      "rewards/rejected": -2.9491138458251953,
      "step": 223,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.499860529986053,
      "grad_norm": 2.936124324798584,
      "learning_rate": 0.00017894983948864431,
      "logits/chosen": -0.5203258991241455,
      "logits/rejected": -0.6186901926994324,
      "logps/chosen": -8.132195472717285,
      "logps/rejected": -41.595703125,
      "loss": 0.7639778256416321,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4688246548175812,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03628700599074364,
      "rewards/margins": 2.6348683834075928,
      "rewards/rejected": -2.598581314086914,
      "step": 224,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.502092050209205,
      "grad_norm": 3.2935783863067627,
      "learning_rate": 0.0001787227261493405,
      "logits/chosen": -0.49744516611099243,
      "logits/rejected": -0.6266440749168396,
      "logps/chosen": -6.2615861892700195,
      "logps/rejected": -52.76634216308594,
      "loss": 0.5845623016357422,
      "memory(GiB)": 46.71,
      "nll_loss": 0.36887243390083313,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10732433944940567,
      "rewards/margins": 3.68102765083313,
      "rewards/rejected": -3.5737037658691406,
      "step": 225,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.5043235704323571,
      "grad_norm": 2.1799988746643066,
      "learning_rate": 0.0001784945399569447,
      "logits/chosen": -0.4298093914985657,
      "logits/rejected": -0.6444836854934692,
      "logps/chosen": -3.285033941268921,
      "logps/rejected": -62.29474639892578,
      "loss": 0.3315008282661438,
      "memory(GiB)": 46.71,
      "nll_loss": 0.21272185444831848,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2050013542175293,
      "rewards/margins": 4.134213924407959,
      "rewards/rejected": -3.9292120933532715,
      "step": 226,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.5065550906555091,
      "grad_norm": 1.7241166830062866,
      "learning_rate": 0.00017826528402123564,
      "logits/chosen": -0.48651599884033203,
      "logits/rejected": -0.6437865495681763,
      "logps/chosen": -2.7627596855163574,
      "logps/rejected": -52.23710632324219,
      "loss": 0.3623155653476715,
      "memory(GiB)": 46.71,
      "nll_loss": 0.25128594040870667,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20487122237682343,
      "rewards/margins": 3.8199470043182373,
      "rewards/rejected": -3.6150758266448975,
      "step": 227,
      "train_speed(iter/s)": 0.011174
    },
    {
      "epoch": 0.5087866108786611,
      "grad_norm": 2.1939537525177,
      "learning_rate": 0.00017803496146657084,
      "logits/chosen": -0.4696894884109497,
      "logits/rejected": -0.648972749710083,
      "logps/chosen": -6.876445770263672,
      "logps/rejected": -58.56797409057617,
      "loss": 0.6069039106369019,
      "memory(GiB)": 46.71,
      "nll_loss": 0.48535749316215515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30394309759140015,
      "rewards/margins": 4.106232643127441,
      "rewards/rejected": -3.8022897243499756,
      "step": 228,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.5110181311018132,
      "grad_norm": 2.2029025554656982,
      "learning_rate": 0.00017780357543184397,
      "logits/chosen": -0.5041614174842834,
      "logits/rejected": -0.6466209292411804,
      "logps/chosen": -6.405285835266113,
      "logps/rejected": -66.68818664550781,
      "loss": 0.530963122844696,
      "memory(GiB)": 46.71,
      "nll_loss": 0.36968475580215454,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1667797863483429,
      "rewards/margins": 5.374972343444824,
      "rewards/rejected": -5.208192825317383,
      "step": 229,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.5132496513249651,
      "grad_norm": 2.9095795154571533,
      "learning_rate": 0.000177571129070442,
      "logits/chosen": -0.5456891655921936,
      "logits/rejected": -0.5758720636367798,
      "logps/chosen": -9.691763877868652,
      "logps/rejected": -49.30378723144531,
      "loss": 0.6552941203117371,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4235466718673706,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05060029774904251,
      "rewards/margins": 3.4885663986206055,
      "rewards/rejected": -3.4379661083221436,
      "step": 230,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.5154811715481171,
      "grad_norm": 2.6140189170837402,
      "learning_rate": 0.00017733762555020234,
      "logits/chosen": -0.526019811630249,
      "logits/rejected": -0.6267409324645996,
      "logps/chosen": -7.562019348144531,
      "logps/rejected": -59.53757858276367,
      "loss": 0.45963698625564575,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3218066692352295,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11903621256351471,
      "rewards/margins": 4.636847019195557,
      "rewards/rejected": -4.517810821533203,
      "step": 231,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.5177126917712692,
      "grad_norm": 7.271316051483154,
      "learning_rate": 0.0001771030680533697,
      "logits/chosen": -0.4977180063724518,
      "logits/rejected": -0.6347447633743286,
      "logps/chosen": -6.1880202293396,
      "logps/rejected": -57.32111358642578,
      "loss": 0.6974989771842957,
      "memory(GiB)": 46.71,
      "nll_loss": 0.41052868962287903,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06481020152568817,
      "rewards/margins": 4.053691864013672,
      "rewards/rejected": -3.988882064819336,
      "step": 232,
      "train_speed(iter/s)": 0.011175
    },
    {
      "epoch": 0.5199442119944212,
      "grad_norm": 6.688237190246582,
      "learning_rate": 0.00017686745977655254,
      "logits/chosen": -0.5404477715492249,
      "logits/rejected": -0.591978907585144,
      "logps/chosen": -7.557736873626709,
      "logps/rejected": -53.3175048828125,
      "loss": 0.6092088222503662,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40488630533218384,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.009284242987632751,
      "rewards/margins": 4.1996588706970215,
      "rewards/rejected": -4.190374851226807,
      "step": 233,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5221757322175732,
      "grad_norm": 2.909499168395996,
      "learning_rate": 0.00017663080393067967,
      "logits/chosen": -0.41550740599632263,
      "logits/rejected": -0.6418639421463013,
      "logps/chosen": -9.175922393798828,
      "logps/rejected": -79.41323852539062,
      "loss": 0.7257620096206665,
      "memory(GiB)": 46.71,
      "nll_loss": 0.47780197858810425,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10455615818500519,
      "rewards/margins": 5.70869255065918,
      "rewards/rejected": -5.6041364669799805,
      "step": 234,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5244072524407253,
      "grad_norm": 24.34648895263672,
      "learning_rate": 0.00017639310374095642,
      "logits/chosen": -0.474945992231369,
      "logits/rejected": -0.6495036482810974,
      "logps/chosen": -9.139482498168945,
      "logps/rejected": -61.76782989501953,
      "loss": 0.7843772172927856,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6071273684501648,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08099109679460526,
      "rewards/margins": 4.556737422943115,
      "rewards/rejected": -4.475746154785156,
      "step": 235,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5266387726638773,
      "grad_norm": 3.9490129947662354,
      "learning_rate": 0.00017615436244682067,
      "logits/chosen": -0.48849430680274963,
      "logits/rejected": -0.6626330614089966,
      "logps/chosen": -6.473770618438721,
      "logps/rejected": -63.68229675292969,
      "loss": 0.7651278972625732,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5939050912857056,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.05846280977129936,
      "rewards/margins": 4.720561981201172,
      "rewards/rejected": -4.662098407745361,
      "step": 236,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5288702928870292,
      "grad_norm": 2.3226640224456787,
      "learning_rate": 0.0001759145833018988,
      "logits/chosen": -0.5129532814025879,
      "logits/rejected": -0.6029571294784546,
      "logps/chosen": -11.16122055053711,
      "logps/rejected": -65.20454406738281,
      "loss": 0.6758726239204407,
      "memory(GiB)": 46.71,
      "nll_loss": 0.44432199001312256,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06612987071275711,
      "rewards/margins": 4.690124034881592,
      "rewards/rejected": -4.62399435043335,
      "step": 237,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5311018131101813,
      "grad_norm": 4.086884498596191,
      "learning_rate": 0.00017567376957396125,
      "logits/chosen": -0.4485701620578766,
      "logits/rejected": -0.6184876561164856,
      "logps/chosen": -9.818546295166016,
      "logps/rejected": -64.69741821289062,
      "loss": 0.841502845287323,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6483633518218994,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15229403972625732,
      "rewards/margins": 4.889117240905762,
      "rewards/rejected": -4.736823558807373,
      "step": 238,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 5.736257076263428,
      "learning_rate": 0.00017543192454487795,
      "logits/chosen": -0.5332171320915222,
      "logits/rejected": -0.6654965877532959,
      "logps/chosen": -8.099287986755371,
      "logps/rejected": -68.04266357421875,
      "loss": 0.6135991215705872,
      "memory(GiB)": 46.71,
      "nll_loss": 0.47898998856544495,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.012953575700521469,
      "rewards/margins": 5.387064456939697,
      "rewards/rejected": -5.374111175537109,
      "step": 239,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5355648535564853,
      "grad_norm": 2.407405138015747,
      "learning_rate": 0.0001751890515105738,
      "logits/chosen": -0.5446051955223083,
      "logits/rejected": -0.6599375009536743,
      "logps/chosen": -5.214753150939941,
      "logps/rejected": -70.1378173828125,
      "loss": 0.6014682054519653,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4095730781555176,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1154915988445282,
      "rewards/margins": 5.451560020446777,
      "rewards/rejected": -5.336068153381348,
      "step": 240,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5377963737796374,
      "grad_norm": 5.806000232696533,
      "learning_rate": 0.00017494515378098347,
      "logits/chosen": -0.5061202049255371,
      "logits/rejected": -0.6401721835136414,
      "logps/chosen": -7.594027996063232,
      "logps/rejected": -72.66096496582031,
      "loss": 0.7293487191200256,
      "memory(GiB)": 46.71,
      "nll_loss": 0.509558379650116,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.20420843362808228,
      "rewards/margins": 5.691208839416504,
      "rewards/rejected": -5.895417213439941,
      "step": 241,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5400278940027894,
      "grad_norm": 20.491920471191406,
      "learning_rate": 0.00017470023468000654,
      "logits/chosen": -0.5332112312316895,
      "logits/rejected": -0.6652044653892517,
      "logps/chosen": -4.244685649871826,
      "logps/rejected": -67.0967788696289,
      "loss": 0.4344191253185272,
      "memory(GiB)": 46.71,
      "nll_loss": 0.27993929386138916,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.039192792028188705,
      "rewards/margins": 5.225668907165527,
      "rewards/rejected": -5.18647575378418,
      "step": 242,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5422594142259414,
      "grad_norm": 1.5459198951721191,
      "learning_rate": 0.00017445429754546209,
      "logits/chosen": -0.4623527526855469,
      "logits/rejected": -0.5880333185195923,
      "logps/chosen": -6.493972301483154,
      "logps/rejected": -61.213016510009766,
      "loss": 0.5002475380897522,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3536112904548645,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23787753283977509,
      "rewards/margins": 4.298797607421875,
      "rewards/rejected": -4.060920715332031,
      "step": 243,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5444909344490935,
      "grad_norm": 10.530829429626465,
      "learning_rate": 0.00017420734572904309,
      "logits/chosen": -0.4680861234664917,
      "logits/rejected": -0.6114482879638672,
      "logps/chosen": -6.2320733070373535,
      "logps/rejected": -51.97254180908203,
      "loss": 0.6590880155563354,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4217822849750519,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1015012115240097,
      "rewards/margins": 3.5804498195648193,
      "rewards/rejected": -3.4789485931396484,
      "step": 244,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5467224546722455,
      "grad_norm": 4.540682792663574,
      "learning_rate": 0.00017395938259627102,
      "logits/chosen": -0.4017217457294464,
      "logits/rejected": -0.5551414489746094,
      "logps/chosen": -4.653961658477783,
      "logps/rejected": -60.12483215332031,
      "loss": 0.417266309261322,
      "memory(GiB)": 46.71,
      "nll_loss": 0.26140686869621277,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2787860631942749,
      "rewards/margins": 3.9614157676696777,
      "rewards/rejected": -3.682629108428955,
      "step": 245,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5489539748953974,
      "grad_norm": 2.821626663208008,
      "learning_rate": 0.00017371041152644976,
      "logits/chosen": -0.46805882453918457,
      "logits/rejected": -0.5824432373046875,
      "logps/chosen": -7.418338775634766,
      "logps/rejected": -49.86051940917969,
      "loss": 0.7901802659034729,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5593172311782837,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2499873787164688,
      "rewards/margins": 3.6353776454925537,
      "rewards/rejected": -3.385390043258667,
      "step": 246,
      "train_speed(iter/s)": 0.011177
    },
    {
      "epoch": 0.5511854951185495,
      "grad_norm": 2.110387086868286,
      "learning_rate": 0.00017346043591261957,
      "logits/chosen": -0.441771000623703,
      "logits/rejected": -0.6270588040351868,
      "logps/chosen": -7.553977012634277,
      "logps/rejected": -58.14564514160156,
      "loss": 0.569815993309021,
      "memory(GiB)": 46.71,
      "nll_loss": 0.40740612149238586,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2955455183982849,
      "rewards/margins": 3.5217554569244385,
      "rewards/rejected": -3.226210117340088,
      "step": 247,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5534170153417015,
      "grad_norm": 2.2252297401428223,
      "learning_rate": 0.0001732094591615109,
      "logits/chosen": -0.3921899199485779,
      "logits/rejected": -0.5801397562026978,
      "logps/chosen": -8.181288719177246,
      "logps/rejected": -63.77415084838867,
      "loss": 0.5467262268066406,
      "memory(GiB)": 46.71,
      "nll_loss": 0.37672924995422363,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2877017557621002,
      "rewards/margins": 4.03120231628418,
      "rewards/rejected": -3.7435007095336914,
      "step": 248,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5556485355648536,
      "grad_norm": 2.368830680847168,
      "learning_rate": 0.00017295748469349804,
      "logits/chosen": -0.4548183083534241,
      "logits/rejected": -0.6428787112236023,
      "logps/chosen": -5.975998878479004,
      "logps/rejected": -64.11628723144531,
      "loss": 0.6057455539703369,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5057395696640015,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24285957217216492,
      "rewards/margins": 4.522610664367676,
      "rewards/rejected": -4.279751300811768,
      "step": 249,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5578800557880056,
      "grad_norm": 5.085129261016846,
      "learning_rate": 0.00017270451594255233,
      "logits/chosen": -0.5124991536140442,
      "logits/rejected": -0.6001466512680054,
      "logps/chosen": -5.997179985046387,
      "logps/rejected": -46.71837615966797,
      "loss": 0.5066366791725159,
      "memory(GiB)": 46.71,
      "nll_loss": 0.31767114996910095,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16393466293811798,
      "rewards/margins": 3.424452066421509,
      "rewards/rejected": -3.260517120361328,
      "step": 250,
      "train_speed(iter/s)": 0.011176
    },
    {
      "epoch": 0.5601115760111576,
      "grad_norm": 3.277879476547241,
      "learning_rate": 0.00017245055635619542,
      "logits/chosen": -0.48801007866859436,
      "logits/rejected": -0.5720742344856262,
      "logps/chosen": -6.175948143005371,
      "logps/rejected": -54.81993865966797,
      "loss": 0.5065844058990479,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3731592297554016,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12712717056274414,
      "rewards/margins": 4.286194801330566,
      "rewards/rejected": -4.159067153930664,
      "step": 251,
      "train_speed(iter/s)": 0.011168
    },
    {
      "epoch": 0.5623430962343097,
      "grad_norm": 6.859767913818359,
      "learning_rate": 0.00017219560939545246,
      "logits/chosen": -0.522152304649353,
      "logits/rejected": -0.5365570187568665,
      "logps/chosen": -14.133954048156738,
      "logps/rejected": -37.61845016479492,
      "loss": 0.8155819177627563,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4886224865913391,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2092638611793518,
      "rewards/margins": 2.255005359649658,
      "rewards/rejected": -2.4642696380615234,
      "step": 252,
      "train_speed(iter/s)": 0.011168
    },
    {
      "epoch": 0.5645746164574617,
      "grad_norm": 8.044095039367676,
      "learning_rate": 0.0001719396785348046,
      "logits/chosen": -0.45501673221588135,
      "logits/rejected": -0.5942042469978333,
      "logps/chosen": -8.521533966064453,
      "logps/rejected": -53.618995666503906,
      "loss": 0.8110232353210449,
      "memory(GiB)": 46.71,
      "nll_loss": 0.42624324560165405,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1203906238079071,
      "rewards/margins": 3.3475189208984375,
      "rewards/rejected": -3.467909574508667,
      "step": 253,
      "train_speed(iter/s)": 0.011168
    },
    {
      "epoch": 0.5668061366806136,
      "grad_norm": 13.483397483825684,
      "learning_rate": 0.000171682767262142,
      "logits/chosen": -0.44267958402633667,
      "logits/rejected": -0.6059361100196838,
      "logps/chosen": -5.3091816902160645,
      "logps/rejected": -49.56216049194336,
      "loss": 0.6120542287826538,
      "memory(GiB)": 46.71,
      "nll_loss": 0.413876473903656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14617875218391418,
      "rewards/margins": 3.480231523513794,
      "rewards/rejected": -3.334052562713623,
      "step": 254,
      "train_speed(iter/s)": 0.011168
    },
    {
      "epoch": 0.5690376569037657,
      "grad_norm": 2.9796292781829834,
      "learning_rate": 0.00017142487907871593,
      "logits/chosen": -0.441107839345932,
      "logits/rejected": -0.5549041628837585,
      "logps/chosen": -6.088810920715332,
      "logps/rejected": -41.22234344482422,
      "loss": 0.556708574295044,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3824915587902069,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2537227272987366,
      "rewards/margins": 2.6484460830688477,
      "rewards/rejected": -2.3947231769561768,
      "step": 255,
      "train_speed(iter/s)": 0.011168
    },
    {
      "epoch": 0.5712691771269177,
      "grad_norm": 1.7595632076263428,
      "learning_rate": 0.00017116601749909142,
      "logits/chosen": -0.4316076636314392,
      "logits/rejected": -0.5781916379928589,
      "logps/chosen": -8.001547813415527,
      "logps/rejected": -45.277442932128906,
      "loss": 0.5357844829559326,
      "memory(GiB)": 46.71,
      "nll_loss": 0.348554790019989,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2612729072570801,
      "rewards/margins": 2.8041718006134033,
      "rewards/rejected": -2.5428988933563232,
      "step": 256,
      "train_speed(iter/s)": 0.011168
    },
    {
      "epoch": 0.5735006973500697,
      "grad_norm": 2.4086761474609375,
      "learning_rate": 0.0001709061860510991,
      "logits/chosen": -0.47363919019699097,
      "logits/rejected": -0.575410008430481,
      "logps/chosen": -6.791253089904785,
      "logps/rejected": -40.982913970947266,
      "loss": 0.5101409554481506,
      "memory(GiB)": 46.71,
      "nll_loss": 0.28215673565864563,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08847678452730179,
      "rewards/margins": 2.6485981941223145,
      "rewards/rejected": -2.5601210594177246,
      "step": 257,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5757322175732218,
      "grad_norm": 2.0793402194976807,
      "learning_rate": 0.00017064538827578723,
      "logits/chosen": -0.4602135419845581,
      "logits/rejected": -0.5832696557044983,
      "logps/chosen": -6.382266044616699,
      "logps/rejected": -46.76750946044922,
      "loss": 0.5784123539924622,
      "memory(GiB)": 46.71,
      "nll_loss": 0.413840115070343,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2313905507326126,
      "rewards/margins": 3.05240797996521,
      "rewards/rejected": -2.821016788482666,
      "step": 258,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5779637377963738,
      "grad_norm": 3.499621629714966,
      "learning_rate": 0.00017038362772737347,
      "logits/chosen": -0.419292688369751,
      "logits/rejected": -0.610460102558136,
      "logps/chosen": -5.339640140533447,
      "logps/rejected": -38.10063171386719,
      "loss": 0.5521044731140137,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3070164620876312,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0932367667555809,
      "rewards/margins": 2.2065370082855225,
      "rewards/rejected": -2.113300323486328,
      "step": 259,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5801952580195258,
      "grad_norm": 2.3477797508239746,
      "learning_rate": 0.00017012090797319628,
      "logits/chosen": -0.46017396450042725,
      "logits/rejected": -0.6195208430290222,
      "logps/chosen": -8.076725006103516,
      "logps/rejected": -50.54127883911133,
      "loss": 0.4606272578239441,
      "memory(GiB)": 46.71,
      "nll_loss": 0.2921941876411438,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22055713832378387,
      "rewards/margins": 3.12892484664917,
      "rewards/rejected": -2.908367872238159,
      "step": 260,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5824267782426779,
      "grad_norm": 2.4851467609405518,
      "learning_rate": 0.0001698572325936665,
      "logits/chosen": -0.4670488238334656,
      "logits/rejected": -0.6289732456207275,
      "logps/chosen": -7.322833061218262,
      "logps/rejected": -50.35585021972656,
      "loss": 0.5902070999145508,
      "memory(GiB)": 46.71,
      "nll_loss": 0.42221367359161377,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1646100878715515,
      "rewards/margins": 3.3369765281677246,
      "rewards/rejected": -3.1723666191101074,
      "step": 261,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5846582984658298,
      "grad_norm": 4.714338779449463,
      "learning_rate": 0.00016959260518221845,
      "logits/chosen": -0.47086095809936523,
      "logits/rejected": -0.5858362913131714,
      "logps/chosen": -8.72909927368164,
      "logps/rejected": -44.19210433959961,
      "loss": 0.6342991590499878,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4329380691051483,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.02014285698533058,
      "rewards/margins": 2.7174415588378906,
      "rewards/rejected": -2.737584114074707,
      "step": 262,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5868898186889818,
      "grad_norm": 1.444083571434021,
      "learning_rate": 0.000169327029345261,
      "logits/chosen": -0.39938443899154663,
      "logits/rejected": -0.6433777809143066,
      "logps/chosen": -3.40937876701355,
      "logps/rejected": -59.414127349853516,
      "loss": 0.3323577046394348,
      "memory(GiB)": 46.71,
      "nll_loss": 0.22196492552757263,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2753269672393799,
      "rewards/margins": 3.9262943267822266,
      "rewards/rejected": -3.650967597961426,
      "step": 263,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5891213389121339,
      "grad_norm": 2.6938703060150146,
      "learning_rate": 0.00016906050870212834,
      "logits/chosen": -0.5094009637832642,
      "logits/rejected": -0.6173782348632812,
      "logps/chosen": -7.775401592254639,
      "logps/rejected": -54.587913513183594,
      "loss": 0.6427381634712219,
      "memory(GiB)": 46.71,
      "nll_loss": 0.46833711862564087,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10746394097805023,
      "rewards/margins": 3.8728220462799072,
      "rewards/rejected": -3.7653582096099854,
      "step": 264,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5913528591352859,
      "grad_norm": 5.664610862731934,
      "learning_rate": 0.00016879304688503086,
      "logits/chosen": -0.4606212079524994,
      "logits/rejected": -0.6145158410072327,
      "logps/chosen": -7.418776512145996,
      "logps/rejected": -52.65889358520508,
      "loss": 0.5809510946273804,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4256901443004608,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11412282288074493,
      "rewards/margins": 3.720710515975952,
      "rewards/rejected": -3.6065876483917236,
      "step": 265,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.5935843793584379,
      "grad_norm": 2.40004825592041,
      "learning_rate": 0.0001685246475390053,
      "logits/chosen": -0.4467337131500244,
      "logits/rejected": -0.6297879219055176,
      "logps/chosen": -8.128316879272461,
      "logps/rejected": -51.549537658691406,
      "loss": 0.6336352229118347,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4354446232318878,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07178286463022232,
      "rewards/margins": 3.265444755554199,
      "rewards/rejected": -3.19366192817688,
      "step": 266,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.59581589958159,
      "grad_norm": 2.6514225006103516,
      "learning_rate": 0.00016825531432186543,
      "logits/chosen": -0.41302669048309326,
      "logits/rejected": -0.5715283751487732,
      "logps/chosen": -6.686316013336182,
      "logps/rejected": -57.61973190307617,
      "loss": 0.4782642722129822,
      "memory(GiB)": 46.71,
      "nll_loss": 0.35745877027511597,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13967037200927734,
      "rewards/margins": 4.3413496017456055,
      "rewards/rejected": -4.201679706573486,
      "step": 267,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.598047419804742,
      "grad_norm": 1.7012206315994263,
      "learning_rate": 0.00016798505090415199,
      "logits/chosen": -0.45638614892959595,
      "logits/rejected": -0.5790466666221619,
      "logps/chosen": -8.776070594787598,
      "logps/rejected": -52.691070556640625,
      "loss": 0.523406982421875,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3796443045139313,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3116152584552765,
      "rewards/margins": 4.0619001388549805,
      "rewards/rejected": -3.750284194946289,
      "step": 268,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.600278940027894,
      "grad_norm": 2.13728666305542,
      "learning_rate": 0.00016771386096908278,
      "logits/chosen": -0.45061424374580383,
      "logits/rejected": -0.6187809109687805,
      "logps/chosen": -6.508578300476074,
      "logps/rejected": -55.61383056640625,
      "loss": 0.5749614238739014,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4280470609664917,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.026448212563991547,
      "rewards/margins": 4.147185802459717,
      "rewards/rejected": -4.1207380294799805,
      "step": 269,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.602510460251046,
      "grad_norm": 2.3225080966949463,
      "learning_rate": 0.00016744174821250237,
      "logits/chosen": -0.411843478679657,
      "logits/rejected": -0.56050044298172,
      "logps/chosen": -11.897720336914062,
      "logps/rejected": -57.52003479003906,
      "loss": 0.796038031578064,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5885191559791565,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.023488737642765045,
      "rewards/margins": 3.5510995388031006,
      "rewards/rejected": -3.5276105403900146,
      "step": 270,
      "train_speed(iter/s)": 0.011169
    },
    {
      "epoch": 0.604741980474198,
      "grad_norm": 11.049079895019531,
      "learning_rate": 0.00016716871634283174,
      "logits/chosen": -0.3955327868461609,
      "logits/rejected": -0.608815610408783,
      "logps/chosen": -4.626939296722412,
      "logps/rejected": -62.783973693847656,
      "loss": 0.44843360781669617,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3314272165298462,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13732756674289703,
      "rewards/margins": 4.360987663269043,
      "rewards/rejected": -4.223659992218018,
      "step": 271,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6069735006973501,
      "grad_norm": 2.914240598678589,
      "learning_rate": 0.00016689476908101782,
      "logits/chosen": -0.38342612981796265,
      "logits/rejected": -0.6166298985481262,
      "logps/chosen": -3.6991920471191406,
      "logps/rejected": -56.026390075683594,
      "loss": 0.5108941793441772,
      "memory(GiB)": 46.71,
      "nll_loss": 0.32679831981658936,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10832832008600235,
      "rewards/margins": 4.004990577697754,
      "rewards/rejected": -3.896662712097168,
      "step": 272,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6092050209205021,
      "grad_norm": 8.852380752563477,
      "learning_rate": 0.00016661991016048275,
      "logits/chosen": -0.5051642060279846,
      "logits/rejected": -0.6240350604057312,
      "logps/chosen": -5.203716278076172,
      "logps/rejected": -55.715755462646484,
      "loss": 0.5184271335601807,
      "memory(GiB)": 46.71,
      "nll_loss": 0.43319958448410034,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.15009382367134094,
      "rewards/margins": 4.303383827209473,
      "rewards/rejected": -4.153290271759033,
      "step": 273,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6114365411436541,
      "grad_norm": 4.726907253265381,
      "learning_rate": 0.00016634414332707297,
      "logits/chosen": -0.4815787672996521,
      "logits/rejected": -0.6398111581802368,
      "logps/chosen": -4.289473056793213,
      "logps/rejected": -61.950286865234375,
      "loss": 0.4551790952682495,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3091404438018799,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.039064548909664154,
      "rewards/margins": 4.923999309539795,
      "rewards/rejected": -4.884934425354004,
      "step": 274,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6136680613668062,
      "grad_norm": 7.542313098907471,
      "learning_rate": 0.00016606747233900815,
      "logits/chosen": -0.44271501898765564,
      "logits/rejected": -0.5747901797294617,
      "logps/chosen": -7.7364325523376465,
      "logps/rejected": -53.02996826171875,
      "loss": 0.744888424873352,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5044885873794556,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.024614796042442322,
      "rewards/margins": 3.7873635292053223,
      "rewards/rejected": -3.811978816986084,
      "step": 275,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6158995815899582,
      "grad_norm": 2.8804731369018555,
      "learning_rate": 0.00016578990096683004,
      "logits/chosen": -0.42431771755218506,
      "logits/rejected": -0.5855906009674072,
      "logps/chosen": -8.4268217086792,
      "logps/rejected": -61.900108337402344,
      "loss": 0.6437846422195435,
      "memory(GiB)": 46.71,
      "nll_loss": 0.46338486671447754,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.010673696175217628,
      "rewards/margins": 4.697712421417236,
      "rewards/rejected": -4.687038898468018,
      "step": 276,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6181311018131102,
      "grad_norm": 16.423973083496094,
      "learning_rate": 0.00016551143299335105,
      "logits/chosen": -0.4473971724510193,
      "logits/rejected": -0.556106448173523,
      "logps/chosen": -8.695633888244629,
      "logps/rejected": -54.007286071777344,
      "loss": 0.7244383096694946,
      "memory(GiB)": 46.71,
      "nll_loss": 0.45499277114868164,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0047096069902181625,
      "rewards/margins": 3.7970499992370605,
      "rewards/rejected": -3.7923407554626465,
      "step": 277,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6203626220362622,
      "grad_norm": 4.081517696380615,
      "learning_rate": 0.00016523207221360266,
      "logits/chosen": -0.3338325619697571,
      "logits/rejected": -0.5568884015083313,
      "logps/chosen": -3.983449697494507,
      "logps/rejected": -66.22798919677734,
      "loss": 0.39685380458831787,
      "memory(GiB)": 46.71,
      "nll_loss": 0.227124884724617,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1147797703742981,
      "rewards/margins": 4.806873321533203,
      "rewards/rejected": -4.692093849182129,
      "step": 278,
      "train_speed(iter/s)": 0.01117
    },
    {
      "epoch": 0.6225941422594142,
      "grad_norm": 1.6026629209518433,
      "learning_rate": 0.0001649518224347838,
      "logits/chosen": -0.45107001066207886,
      "logits/rejected": -0.6225546002388,
      "logps/chosen": -2.7264139652252197,
      "logps/rejected": -68.20640563964844,
      "loss": 0.2848789095878601,
      "memory(GiB)": 46.71,
      "nll_loss": 0.1853371113538742,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16342315077781677,
      "rewards/margins": 5.450206756591797,
      "rewards/rejected": -5.2867841720581055,
      "step": 279,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.6248256624825662,
      "grad_norm": 2.1419737339019775,
      "learning_rate": 0.0001646706874762089,
      "logits/chosen": -0.4340333938598633,
      "logits/rejected": -0.6073229312896729,
      "logps/chosen": -4.69732141494751,
      "logps/rejected": -70.4720687866211,
      "loss": 0.42912930250167847,
      "memory(GiB)": 46.71,
      "nll_loss": 0.33446142077445984,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18399639427661896,
      "rewards/margins": 5.338755130767822,
      "rewards/rejected": -5.154758453369141,
      "step": 280,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.6270571827057183,
      "grad_norm": 4.532802581787109,
      "learning_rate": 0.0001643886711692557,
      "logits/chosen": -0.4100894331932068,
      "logits/rejected": -0.5919958353042603,
      "logps/chosen": -4.708090782165527,
      "logps/rejected": -55.577186584472656,
      "loss": 0.5581774711608887,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4139384627342224,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19322583079338074,
      "rewards/margins": 3.968625068664551,
      "rewards/rejected": -3.7753994464874268,
      "step": 281,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.6292887029288703,
      "grad_norm": 2.134107828140259,
      "learning_rate": 0.00016410577735731345,
      "logits/chosen": -0.4810371994972229,
      "logits/rejected": -0.6207767128944397,
      "logps/chosen": -6.703675746917725,
      "logps/rejected": -54.464988708496094,
      "loss": 0.6276415586471558,
      "memory(GiB)": 46.71,
      "nll_loss": 0.47040659189224243,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.06548688560724258,
      "rewards/margins": 3.9635109901428223,
      "rewards/rejected": -3.898024320602417,
      "step": 282,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.6315202231520223,
      "grad_norm": 2.135093927383423,
      "learning_rate": 0.00016382200989572998,
      "logits/chosen": -0.5298405885696411,
      "logits/rejected": -0.6139633059501648,
      "logps/chosen": -8.422263145446777,
      "logps/rejected": -48.18453598022461,
      "loss": 0.7457632422447205,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5003448128700256,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06589431315660477,
      "rewards/margins": 3.5135788917541504,
      "rewards/rejected": -3.4476842880249023,
      "step": 283,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.6337517433751744,
      "grad_norm": 4.251437664031982,
      "learning_rate": 0.00016353737265175963,
      "logits/chosen": -0.5017900466918945,
      "logits/rejected": -0.6134066581726074,
      "logps/chosen": -9.804008483886719,
      "logps/rejected": -52.773406982421875,
      "loss": 0.8265900611877441,
      "memory(GiB)": 46.71,
      "nll_loss": 0.6167674660682678,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04522796720266342,
      "rewards/margins": 3.658693552017212,
      "rewards/rejected": -3.6134657859802246,
      "step": 284,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.6359832635983264,
      "grad_norm": 10.930916786193848,
      "learning_rate": 0.00016325186950451022,
      "logits/chosen": -0.5018693804740906,
      "logits/rejected": -0.5625038146972656,
      "logps/chosen": -6.777836799621582,
      "logps/rejected": -42.862220764160156,
      "loss": 0.7121441960334778,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4460356831550598,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.041172679513692856,
      "rewards/margins": 3.1301026344299316,
      "rewards/rejected": -3.0889296531677246,
      "step": 285,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6382147838214783,
      "grad_norm": 2.1487908363342285,
      "learning_rate": 0.00016296550434489036,
      "logits/chosen": -0.47381335496902466,
      "logits/rejected": -0.6210272312164307,
      "logps/chosen": -7.876993179321289,
      "logps/rejected": -67.23505401611328,
      "loss": 0.3986350893974304,
      "memory(GiB)": 46.71,
      "nll_loss": 0.31996262073516846,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.07089027762413025,
      "rewards/margins": 4.680482864379883,
      "rewards/rejected": -4.751373291015625,
      "step": 286,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6404463040446304,
      "grad_norm": 7.843413829803467,
      "learning_rate": 0.0001626782810755565,
      "logits/chosen": -0.3885619044303894,
      "logits/rejected": -0.5521731972694397,
      "logps/chosen": -6.2263922691345215,
      "logps/rejected": -72.29328155517578,
      "loss": 0.608590304851532,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4712628126144409,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1404297947883606,
      "rewards/margins": 4.971117973327637,
      "rewards/rejected": -4.8306884765625,
      "step": 287,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6426778242677824,
      "grad_norm": 2.2685890197753906,
      "learning_rate": 0.00016239020361085945,
      "logits/chosen": -0.3876466155052185,
      "logits/rejected": -0.5563790202140808,
      "logps/chosen": -6.572446823120117,
      "logps/rejected": -58.97338104248047,
      "loss": 0.5379603505134583,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3794347047805786,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.163069948554039,
      "rewards/margins": 4.21791934967041,
      "rewards/rejected": -4.054849624633789,
      "step": 288,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6449093444909344,
      "grad_norm": 2.607456922531128,
      "learning_rate": 0.0001621012758767913,
      "logits/chosen": -0.40551966428756714,
      "logits/rejected": -0.5489031672477722,
      "logps/chosen": -5.411971569061279,
      "logps/rejected": -56.15552520751953,
      "loss": 0.5132619738578796,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3366192579269409,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16010434925556183,
      "rewards/margins": 3.999542713165283,
      "rewards/rejected": -3.8394384384155273,
      "step": 289,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6471408647140865,
      "grad_norm": 2.2857182025909424,
      "learning_rate": 0.0001618115018109318,
      "logits/chosen": -0.4304552376270294,
      "logits/rejected": -0.602503776550293,
      "logps/chosen": -4.60065221786499,
      "logps/rejected": -54.26784133911133,
      "loss": 0.5486267805099487,
      "memory(GiB)": 46.71,
      "nll_loss": 0.37609028816223145,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08251349627971649,
      "rewards/margins": 3.8639562129974365,
      "rewards/rejected": -3.781442165374756,
      "step": 290,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6493723849372385,
      "grad_norm": 1.324331283569336,
      "learning_rate": 0.0001615208853623947,
      "logits/chosen": -0.46335089206695557,
      "logits/rejected": -0.597943127155304,
      "logps/chosen": -3.3643994331359863,
      "logps/rejected": -59.1411247253418,
      "loss": 0.34732431173324585,
      "memory(GiB)": 46.71,
      "nll_loss": 0.19164429605007172,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.28088483214378357,
      "rewards/margins": 4.684600830078125,
      "rewards/rejected": -4.403716087341309,
      "step": 291,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6516039051603905,
      "grad_norm": 3.25237774848938,
      "learning_rate": 0.00016122943049177405,
      "logits/chosen": -0.4758291244506836,
      "logits/rejected": -0.5325515866279602,
      "logps/chosen": -13.164379119873047,
      "logps/rejected": -49.068603515625,
      "loss": 0.7119560837745667,
      "memory(GiB)": 46.71,
      "nll_loss": 0.4212368130683899,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.031183507293462753,
      "rewards/margins": 3.5282320976257324,
      "rewards/rejected": -3.4970486164093018,
      "step": 292,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6538354253835426,
      "grad_norm": 1.9222995042800903,
      "learning_rate": 0.00016093714117109,
      "logits/chosen": -0.42248421907424927,
      "logits/rejected": -0.6226801872253418,
      "logps/chosen": -5.316409111022949,
      "logps/rejected": -63.825557708740234,
      "loss": 0.5835841298103333,
      "memory(GiB)": 46.71,
      "nll_loss": 0.46457451581954956,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2873809039592743,
      "rewards/margins": 4.637524604797363,
      "rewards/rejected": -4.3501434326171875,
      "step": 293,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6560669456066945,
      "grad_norm": 2.3776681423187256,
      "learning_rate": 0.00016064402138373489,
      "logits/chosen": -0.399809867143631,
      "logits/rejected": -0.5734027028083801,
      "logps/chosen": -5.731583118438721,
      "logps/rejected": -65.56729125976562,
      "loss": 0.5586244463920593,
      "memory(GiB)": 46.71,
      "nll_loss": 0.41006267070770264,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.09848666191101074,
      "rewards/margins": 4.583428382873535,
      "rewards/rejected": -4.4849419593811035,
      "step": 294,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6582984658298466,
      "grad_norm": 2.861478567123413,
      "learning_rate": 0.0001603500751244188,
      "logits/chosen": -0.466978520154953,
      "logits/rejected": -0.5577737092971802,
      "logps/chosen": -10.857172966003418,
      "logps/rejected": -45.819541931152344,
      "loss": 0.6700180768966675,
      "memory(GiB)": 46.71,
      "nll_loss": 0.44924986362457275,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.08089883625507355,
      "rewards/margins": 3.153740882873535,
      "rewards/rejected": -3.2346394062042236,
      "step": 295,
      "train_speed(iter/s)": 0.011172
    },
    {
      "epoch": 0.6605299860529986,
      "grad_norm": 2.3550102710723877,
      "learning_rate": 0.00016005530639911524,
      "logits/chosen": -0.42725443840026855,
      "logits/rejected": -0.6156260967254639,
      "logps/chosen": -4.741040229797363,
      "logps/rejected": -73.73209381103516,
      "loss": 0.41657161712646484,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3108736276626587,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20638638734817505,
      "rewards/margins": 5.785445690155029,
      "rewards/rejected": -5.57905912399292,
      "step": 296,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.6627615062761506,
      "grad_norm": 2.7637619972229004,
      "learning_rate": 0.00015975971922500644,
      "logits/chosen": -0.4365236759185791,
      "logits/rejected": -0.5944050550460815,
      "logps/chosen": -5.906551837921143,
      "logps/rejected": -64.35894775390625,
      "loss": 0.5716227293014526,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3685966730117798,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14344872534275055,
      "rewards/margins": 4.830230236053467,
      "rewards/rejected": -4.68678092956543,
      "step": 297,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.6649930264993027,
      "grad_norm": 3.1376020908355713,
      "learning_rate": 0.00015946331763042867,
      "logits/chosen": -0.43532899022102356,
      "logits/rejected": -0.5989662408828735,
      "logps/chosen": -6.79918909072876,
      "logps/rejected": -50.513153076171875,
      "loss": 0.552319347858429,
      "memory(GiB)": 46.71,
      "nll_loss": 0.3487764298915863,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16602559387683868,
      "rewards/margins": 3.815488338470459,
      "rewards/rejected": -3.6494626998901367,
      "step": 298,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.6672245467224547,
      "grad_norm": 4.621730804443359,
      "learning_rate": 0.00015916610565481736,
      "logits/chosen": -0.46591538190841675,
      "logits/rejected": -0.653298020362854,
      "logps/chosen": -2.609394073486328,
      "logps/rejected": -72.14924621582031,
      "loss": 0.33943986892700195,
      "memory(GiB)": 46.71,
      "nll_loss": 0.26801469922065735,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17833705246448517,
      "rewards/margins": 5.548633098602295,
      "rewards/rejected": -5.370296478271484,
      "step": 299,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.6694560669456067,
      "grad_norm": 3.571917772293091,
      "learning_rate": 0.00015886808734865202,
      "logits/chosen": -0.4337051510810852,
      "logits/rejected": -0.5879015922546387,
      "logps/chosen": -6.756601333618164,
      "logps/rejected": -48.003501892089844,
      "loss": 0.7254310846328735,
      "memory(GiB)": 46.71,
      "nll_loss": 0.5357213616371155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1567414551973343,
      "rewards/margins": 3.4603137969970703,
      "rewards/rejected": -3.303572654724121,
      "step": 300,
      "train_speed(iter/s)": 0.011173
    },
    {
      "epoch": 0.6694560669456067,
      "eval_logits/chosen": -0.4272937476634979,
      "eval_logits/rejected": -0.5822427272796631,
      "eval_logps/chosen": -7.934236526489258,
      "eval_logps/rejected": -61.74112319946289,
      "eval_loss": 0.603725254535675,
      "eval_nll_loss": 0.4224081039428711,
      "eval_rewards/accuracies": 0.9166666865348816,
      "eval_rewards/chosen": 0.0887039378285408,
      "eval_rewards/margins": 4.4511003494262695,
      "eval_rewards/rejected": -4.362396717071533,
      "eval_runtime": 157.1594,
      "eval_samples_per_second": 0.916,
      "eval_steps_per_second": 0.458,
      "step": 300
    }
  ],
  "logging_steps": 1,
  "max_steps": 896,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.6529592425434317e+17,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}