{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.6287331027978623,
  "eval_steps": 300,
  "global_step": 250,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002514932411191449,
      "grad_norm": 5.559185028076172,
      "learning_rate": 3.3333333333333333e-06,
      "logits/chosen": -0.5158984065055847,
      "logits/rejected": -0.6433685421943665,
      "logps/chosen": -8.879493713378906,
      "logps/rejected": -18.298219680786133,
      "loss": 1.2430726289749146,
      "memory(GiB)": 30.69,
      "nll_loss": 0.5499253869056702,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.010477
    },
    {
      "epoch": 0.005029864822382898,
      "grad_norm": 5.400235176086426,
      "learning_rate": 6.666666666666667e-06,
      "logits/chosen": -0.5219721794128418,
      "logits/rejected": -0.6263731718063354,
      "logps/chosen": -4.77984094619751,
      "logps/rejected": -15.150838851928711,
      "loss": 1.0675525665283203,
      "memory(GiB)": 33.9,
      "nll_loss": 0.3744054436683655,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.010866
    },
    {
      "epoch": 0.007544797233574348,
      "grad_norm": 7.852550983428955,
      "learning_rate": 1e-05,
      "logits/chosen": -0.48881009221076965,
      "logits/rejected": -0.6540625691413879,
      "logps/chosen": -4.714770793914795,
      "logps/rejected": -17.99374008178711,
      "loss": 1.0851404666900635,
      "memory(GiB)": 33.9,
      "nll_loss": 0.3894187808036804,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002094469266012311,
      "rewards/margins": -0.004971538204699755,
      "rewards/rejected": 0.0028770684730261564,
      "step": 3,
      "train_speed(iter/s)": 0.011002
    },
    {
      "epoch": 0.010059729644765796,
      "grad_norm": 10.150762557983398,
      "learning_rate": 1.3333333333333333e-05,
      "logits/chosen": -0.4506370425224304,
      "logits/rejected": -0.6422093510627747,
      "logps/chosen": -3.6849846839904785,
      "logps/rejected": -21.201997756958008,
      "loss": 1.092491865158081,
      "memory(GiB)": 37.36,
      "nll_loss": 0.4020901322364807,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007022961974143982,
      "rewards/margins": 0.0057290042750537395,
      "rewards/rejected": 0.001293957349844277,
      "step": 4,
      "train_speed(iter/s)": 0.011051
    },
    {
      "epoch": 0.012574662055957246,
      "grad_norm": 6.588510036468506,
      "learning_rate": 1.6666666666666667e-05,
      "logits/chosen": -0.4159216284751892,
      "logits/rejected": -0.5884239673614502,
      "logps/chosen": -8.611150741577148,
      "logps/rejected": -19.931533813476562,
      "loss": 1.1986628770828247,
      "memory(GiB)": 37.36,
      "nll_loss": 0.5155895948410034,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.005048489198088646,
      "rewards/margins": 0.020845137536525726,
      "rewards/rejected": -0.01579664833843708,
      "step": 5,
      "train_speed(iter/s)": 0.011086
    },
    {
      "epoch": 0.015089594467148696,
      "grad_norm": 3.965754508972168,
      "learning_rate": 2e-05,
      "logits/chosen": -0.47118157148361206,
      "logits/rejected": -0.5790206789970398,
      "logps/chosen": -9.572708129882812,
      "logps/rejected": -19.066829681396484,
      "loss": 1.1965497732162476,
      "memory(GiB)": 37.36,
      "nll_loss": 0.5279884338378906,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.006116065196692944,
      "rewards/margins": 0.05174415558576584,
      "rewards/rejected": -0.04562808945775032,
      "step": 6,
      "train_speed(iter/s)": 0.01111
    },
    {
      "epoch": 0.017604526878340146,
      "grad_norm": 2.719625234603882,
      "learning_rate": 2.3333333333333336e-05,
      "logits/chosen": -0.40740644931793213,
      "logits/rejected": -0.5419386029243469,
      "logps/chosen": -9.413152694702148,
      "logps/rejected": -24.703401565551758,
      "loss": 1.1549676656723022,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5149396061897278,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02250390499830246,
      "rewards/margins": 0.12123934924602509,
      "rewards/rejected": -0.14374326169490814,
      "step": 7,
      "train_speed(iter/s)": 0.011114
    },
    {
      "epoch": 0.020119459289531592,
      "grad_norm": 3.325852394104004,
      "learning_rate": 2.6666666666666667e-05,
      "logits/chosen": -0.4091758728027344,
      "logits/rejected": -0.5692812204360962,
      "logps/chosen": -5.904629230499268,
      "logps/rejected": -20.752662658691406,
      "loss": 1.0213468074798584,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3945533037185669,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.041384391486644745,
      "rewards/margins": 0.17802417278289795,
      "rewards/rejected": -0.2194085717201233,
      "step": 8,
      "train_speed(iter/s)": 0.011127
    },
    {
      "epoch": 0.022634391700723042,
      "grad_norm": 3.5531411170959473,
      "learning_rate": 3e-05,
      "logits/chosen": -0.3281947374343872,
      "logits/rejected": -0.5156837701797485,
      "logps/chosen": -5.975313186645508,
      "logps/rejected": -27.066158294677734,
      "loss": 0.8695712089538574,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3611774742603302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0457351878285408,
      "rewards/margins": 0.4895857870578766,
      "rewards/rejected": -0.443850576877594,
      "step": 9,
      "train_speed(iter/s)": 0.011136
    },
    {
      "epoch": 0.025149324111914492,
      "grad_norm": 4.356088161468506,
      "learning_rate": 3.3333333333333335e-05,
      "logits/chosen": -0.43971675634384155,
      "logits/rejected": -0.5702978372573853,
      "logps/chosen": -6.376318454742432,
      "logps/rejected": -19.42030143737793,
      "loss": 1.0046417713165283,
      "memory(GiB)": 40.85,
      "nll_loss": 0.42133060097694397,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.17984721064567566,
      "rewards/margins": 0.43266722559928894,
      "rewards/rejected": -0.6125144362449646,
      "step": 10,
      "train_speed(iter/s)": 0.01115
    },
    {
      "epoch": 0.027664256523105942,
      "grad_norm": 3.9732213020324707,
      "learning_rate": 3.6666666666666666e-05,
      "logits/chosen": -0.4561832845211029,
      "logits/rejected": -0.6065505146980286,
      "logps/chosen": -6.498957633972168,
      "logps/rejected": -27.428552627563477,
      "loss": 0.9565106630325317,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40273764729499817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12391924858093262,
      "rewards/margins": 0.5706068277359009,
      "rewards/rejected": -0.6945260763168335,
      "step": 11,
      "train_speed(iter/s)": 0.011161
    },
    {
      "epoch": 0.030179188934297392,
      "grad_norm": 2.2540578842163086,
      "learning_rate": 4e-05,
      "logits/chosen": -0.42088037729263306,
      "logits/rejected": -0.6009025573730469,
      "logps/chosen": -4.243446350097656,
      "logps/rejected": -26.567686080932617,
      "loss": 0.7599425911903381,
      "memory(GiB)": 40.85,
      "nll_loss": 0.28007903695106506,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.14110325276851654,
      "rewards/margins": 0.6590132713317871,
      "rewards/rejected": -0.8001165390014648,
      "step": 12,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.03269412134548884,
      "grad_norm": 6.460833549499512,
      "learning_rate": 4.3333333333333334e-05,
      "logits/chosen": -0.4136754274368286,
      "logits/rejected": -0.5452494621276855,
      "logps/chosen": -7.585506439208984,
      "logps/rejected": -21.19340705871582,
      "loss": 1.223608136177063,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6720148921012878,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.25483113527297974,
      "rewards/margins": 0.49035438895225525,
      "rewards/rejected": -0.7451854944229126,
      "step": 13,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.03520905375668029,
      "grad_norm": 4.092947006225586,
      "learning_rate": 4.666666666666667e-05,
      "logits/chosen": -0.527854323387146,
      "logits/rejected": -0.5994939804077148,
      "logps/chosen": -8.185604095458984,
      "logps/rejected": -17.946279525756836,
      "loss": 1.2557107210159302,
      "memory(GiB)": 40.85,
      "nll_loss": 0.66472989320755,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.145107239484787,
      "rewards/margins": 0.41058170795440674,
      "rewards/rejected": -0.5556889772415161,
      "step": 14,
      "train_speed(iter/s)": 0.011194
    },
    {
      "epoch": 0.03772398616787174,
      "grad_norm": 3.926342248916626,
      "learning_rate": 5e-05,
      "logits/chosen": -0.407508909702301,
      "logits/rejected": -0.5949459075927734,
      "logps/chosen": -3.4148104190826416,
      "logps/rejected": -22.69104766845703,
      "loss": 0.8548452258110046,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40506720542907715,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.025914989411830902,
      "rewards/margins": 0.6952927708625793,
      "rewards/rejected": -0.7212077379226685,
      "step": 15,
      "train_speed(iter/s)": 0.0112
    },
    {
      "epoch": 0.040238918579063185,
      "grad_norm": 2.4273853302001953,
      "learning_rate": 5.333333333333333e-05,
      "logits/chosen": -0.3462049961090088,
      "logits/rejected": -0.58342045545578,
      "logps/chosen": -3.8651039600372314,
      "logps/rejected": -28.470134735107422,
      "loss": 0.821665346622467,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3309793472290039,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04178892821073532,
      "rewards/margins": 0.5164157152175903,
      "rewards/rejected": -0.4746267795562744,
      "step": 16,
      "train_speed(iter/s)": 0.011202
    },
    {
      "epoch": 0.04275385099025464,
      "grad_norm": 1.8921568393707275,
      "learning_rate": 5.666666666666667e-05,
      "logits/chosen": -0.5054872632026672,
      "logits/rejected": -0.6153637170791626,
      "logps/chosen": -1.8910857439041138,
      "logps/rejected": -16.781475067138672,
      "loss": 0.7665270566940308,
      "memory(GiB)": 40.85,
      "nll_loss": 0.2412014603614807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04062865674495697,
      "rewards/margins": 0.41468262672424316,
      "rewards/rejected": -0.3740540146827698,
      "step": 17,
      "train_speed(iter/s)": 0.011212
    },
    {
      "epoch": 0.045268783401446085,
      "grad_norm": 3.2724339962005615,
      "learning_rate": 6e-05,
      "logits/chosen": -0.45994284749031067,
      "logits/rejected": -0.5327339768409729,
      "logps/chosen": -8.705971717834473,
      "logps/rejected": -17.33376121520996,
      "loss": 1.2050995826721191,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5643416047096252,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.048418305814266205,
      "rewards/margins": 0.16582812368869781,
      "rewards/rejected": -0.21424642205238342,
      "step": 18,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.04778371581263754,
      "grad_norm": 2.6650567054748535,
      "learning_rate": 6.333333333333333e-05,
      "logits/chosen": -0.48406994342803955,
      "logits/rejected": -0.5535129308700562,
      "logps/chosen": -10.087296485900879,
      "logps/rejected": -17.584102630615234,
      "loss": 1.1602699756622314,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5509804487228394,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005405411124229431,
      "rewards/margins": 0.23162491619586945,
      "rewards/rejected": -0.23703034222126007,
      "step": 19,
      "train_speed(iter/s)": 0.01122
    },
    {
      "epoch": 0.050298648223828984,
      "grad_norm": 2.393120527267456,
      "learning_rate": 6.666666666666667e-05,
      "logits/chosen": -0.4699333906173706,
      "logits/rejected": -0.5745525360107422,
      "logps/chosen": -5.482749938964844,
      "logps/rejected": -15.294036865234375,
      "loss": 1.0463979244232178,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40886539220809937,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.030414806678891182,
      "rewards/margins": 0.18470264971256256,
      "rewards/rejected": -0.15428784489631653,
      "step": 20,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.05281358063502043,
      "grad_norm": 1.7328065633773804,
      "learning_rate": 7e-05,
      "logits/chosen": -0.5130109190940857,
      "logits/rejected": -0.6171243190765381,
      "logps/chosen": -8.619268417358398,
      "logps/rejected": -18.43971824645996,
      "loss": 1.1801689863204956,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6120539903640747,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07324957102537155,
      "rewards/margins": 0.3033149242401123,
      "rewards/rejected": -0.23006536066532135,
      "step": 21,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.055328513046211884,
      "grad_norm": 2.848578691482544,
      "learning_rate": 7.333333333333333e-05,
      "logits/chosen": -0.46683233976364136,
      "logits/rejected": -0.5329924821853638,
      "logps/chosen": -4.357663154602051,
      "logps/rejected": -15.35399055480957,
      "loss": 0.9406945705413818,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3216635584831238,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.010179778560996056,
      "rewards/margins": 0.2008945494890213,
      "rewards/rejected": -0.21107429265975952,
      "step": 22,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.05784344545740333,
      "grad_norm": 1.7426271438598633,
      "learning_rate": 7.666666666666667e-05,
      "logits/chosen": -0.5002316236495972,
      "logits/rejected": -0.5488481521606445,
      "logps/chosen": -5.690494537353516,
      "logps/rejected": -15.590461730957031,
      "loss": 0.9775924682617188,
      "memory(GiB)": 40.85,
      "nll_loss": 0.355554461479187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.009325886145234108,
      "rewards/margins": 0.18430516123771667,
      "rewards/rejected": -0.1749793142080307,
      "step": 23,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.060358377868594784,
      "grad_norm": 1.8518409729003906,
      "learning_rate": 8e-05,
      "logits/chosen": -0.5258264541625977,
      "logits/rejected": -0.5805689096450806,
      "logps/chosen": -9.133979797363281,
      "logps/rejected": -17.886051177978516,
      "loss": 1.1810051202774048,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6029603481292725,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.04269062727689743,
      "rewards/margins": 0.2787190079689026,
      "rewards/rejected": -0.23602835834026337,
      "step": 24,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.06287331027978624,
      "grad_norm": 2.4842066764831543,
      "learning_rate": 8.333333333333334e-05,
      "logits/chosen": -0.44220152497291565,
      "logits/rejected": -0.5788168907165527,
      "logps/chosen": -4.989194869995117,
      "logps/rejected": -18.047056198120117,
      "loss": 1.0011777877807617,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4066605269908905,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.021916719153523445,
      "rewards/margins": 0.2587287425994873,
      "rewards/rejected": -0.236812025308609,
      "step": 25,
      "train_speed(iter/s)": 0.011239
    },
    {
      "epoch": 0.06538824269097768,
      "grad_norm": 1.766322135925293,
      "learning_rate": 8.666666666666667e-05,
      "logits/chosen": -0.3389984369277954,
      "logits/rejected": -0.543292224407196,
      "logps/chosen": -11.735610961914062,
      "logps/rejected": -26.25033950805664,
      "loss": 1.0063844919204712,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5172990560531616,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07315510511398315,
      "rewards/margins": 0.5734083652496338,
      "rewards/rejected": -0.5002533197402954,
      "step": 26,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.06790317510216913,
      "grad_norm": 3.3314130306243896,
      "learning_rate": 9e-05,
      "logits/chosen": -0.37741950154304504,
      "logits/rejected": -0.5884826183319092,
      "logps/chosen": -5.442512035369873,
      "logps/rejected": -27.738323211669922,
      "loss": 0.8821365237236023,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4055330753326416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03083166666328907,
      "rewards/margins": 0.6056570410728455,
      "rewards/rejected": -0.6364887952804565,
      "step": 27,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.07041810751336058,
      "grad_norm": 2.849647283554077,
      "learning_rate": 9.333333333333334e-05,
      "logits/chosen": -0.4310716688632965,
      "logits/rejected": -0.5529810190200806,
      "logps/chosen": -8.224971771240234,
      "logps/rejected": -26.79846954345703,
      "loss": 0.8747767806053162,
      "memory(GiB)": 40.85,
      "nll_loss": 0.45313140749931335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07478700578212738,
      "rewards/margins": 0.8052245378494263,
      "rewards/rejected": -0.8800115585327148,
      "step": 28,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.07293303992455202,
      "grad_norm": 2.247368097305298,
      "learning_rate": 9.666666666666667e-05,
      "logits/chosen": -0.37296879291534424,
      "logits/rejected": -0.5259015560150146,
      "logps/chosen": -6.546780109405518,
      "logps/rejected": -25.603477478027344,
      "loss": 0.9804578423500061,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4873877465724945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.046319298446178436,
      "rewards/margins": 0.5485984086990356,
      "rewards/rejected": -0.5949177145957947,
      "step": 29,
      "train_speed(iter/s)": 0.011239
    },
    {
      "epoch": 0.07544797233574348,
      "grad_norm": 1.7451585531234741,
      "learning_rate": 0.0001,
      "logits/chosen": -0.36992424726486206,
      "logits/rejected": -0.45540890097618103,
      "logps/chosen": -6.483048439025879,
      "logps/rejected": -18.744525909423828,
      "loss": 0.9453181624412537,
      "memory(GiB)": 40.85,
      "nll_loss": 0.37866073846817017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0520947091281414,
      "rewards/margins": 0.33587026596069336,
      "rewards/rejected": -0.28377556800842285,
      "step": 30,
      "train_speed(iter/s)": 0.01124
    },
    {
      "epoch": 0.07796290474693493,
      "grad_norm": 2.8843863010406494,
      "learning_rate": 0.00010333333333333334,
      "logits/chosen": -0.3973531723022461,
      "logits/rejected": -0.4880366325378418,
      "logps/chosen": -6.5046491622924805,
      "logps/rejected": -18.793209075927734,
      "loss": 1.0352380275726318,
      "memory(GiB)": 40.85,
      "nll_loss": 0.514582097530365,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06333249807357788,
      "rewards/margins": 0.5268359780311584,
      "rewards/rejected": -0.4635034203529358,
      "step": 31,
      "train_speed(iter/s)": 0.011242
    },
    {
      "epoch": 0.08047783715812637,
      "grad_norm": 1.9934616088867188,
      "learning_rate": 0.00010666666666666667,
      "logits/chosen": -0.38435813784599304,
      "logits/rejected": -0.4866703450679779,
      "logps/chosen": -5.984093189239502,
      "logps/rejected": -28.37721061706543,
      "loss": 0.9228850603103638,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3919341564178467,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08762034773826599,
      "rewards/margins": 0.4990919232368469,
      "rewards/rejected": -0.41147157549858093,
      "step": 32,
      "train_speed(iter/s)": 0.011242
    },
    {
      "epoch": 0.08299276956931782,
      "grad_norm": 2.605156421661377,
      "learning_rate": 0.00011000000000000002,
      "logits/chosen": -0.3683161437511444,
      "logits/rejected": -0.4800977110862732,
      "logps/chosen": -6.79887056350708,
      "logps/rejected": -18.521530151367188,
      "loss": 0.9287618398666382,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4155515730381012,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.062238909304142,
      "rewards/margins": 0.4814634323120117,
      "rewards/rejected": -0.41922450065612793,
      "step": 33,
      "train_speed(iter/s)": 0.011243
    },
    {
      "epoch": 0.08550770198050928,
      "grad_norm": 2.1593198776245117,
      "learning_rate": 0.00011333333333333334,
      "logits/chosen": -0.3462904989719391,
      "logits/rejected": -0.4482634663581848,
      "logps/chosen": -4.484028339385986,
      "logps/rejected": -22.155223846435547,
      "loss": 0.8287199139595032,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3564324378967285,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.011370105668902397,
      "rewards/margins": 0.8015316724777222,
      "rewards/rejected": -0.8129018545150757,
      "step": 34,
      "train_speed(iter/s)": 0.011244
    },
    {
      "epoch": 0.08802263439170073,
      "grad_norm": 2.1223862171173096,
      "learning_rate": 0.00011666666666666668,
      "logits/chosen": -0.23746490478515625,
      "logits/rejected": -0.4123544991016388,
      "logps/chosen": -4.062009334564209,
      "logps/rejected": -30.214771270751953,
      "loss": 0.8381334543228149,
      "memory(GiB)": 40.85,
      "nll_loss": 0.45768678188323975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06588438153266907,
      "rewards/margins": 1.1650195121765137,
      "rewards/rejected": -1.0991352796554565,
      "step": 35,
      "train_speed(iter/s)": 0.011244
    },
    {
      "epoch": 0.09053756680289217,
      "grad_norm": 1.7519793510437012,
      "learning_rate": 0.00012,
      "logits/chosen": -0.274143248796463,
      "logits/rejected": -0.4072067439556122,
      "logps/chosen": -7.12678861618042,
      "logps/rejected": -29.586929321289062,
      "loss": 0.7818687558174133,
      "memory(GiB)": 40.85,
      "nll_loss": 0.38336753845214844,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.030900314450263977,
      "rewards/margins": 1.0601840019226074,
      "rewards/rejected": -1.0292836427688599,
      "step": 36,
      "train_speed(iter/s)": 0.011244
    },
    {
      "epoch": 0.09305249921408362,
      "grad_norm": 2.2621781826019287,
      "learning_rate": 0.00012333333333333334,
      "logits/chosen": -0.30612003803253174,
      "logits/rejected": -0.44930776953697205,
      "logps/chosen": -5.637537956237793,
      "logps/rejected": -30.86564064025879,
      "loss": 0.7518173456192017,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4073091149330139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1094760149717331,
      "rewards/margins": 1.2509604692459106,
      "rewards/rejected": -1.3604364395141602,
      "step": 37,
      "train_speed(iter/s)": 0.011245
    },
    {
      "epoch": 0.09556743162527508,
      "grad_norm": 2.6005046367645264,
      "learning_rate": 0.00012666666666666666,
      "logits/chosen": -0.36841830611228943,
      "logits/rejected": -0.44501256942749023,
      "logps/chosen": -8.786764144897461,
      "logps/rejected": -21.693777084350586,
      "loss": 0.967311441898346,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5350124835968018,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05166015028953552,
      "rewards/margins": 0.9184513092041016,
      "rewards/rejected": -0.9701113700866699,
      "step": 38,
      "train_speed(iter/s)": 0.011247
    },
    {
      "epoch": 0.09808236403646652,
      "grad_norm": 7.679723262786865,
      "learning_rate": 0.00013000000000000002,
      "logits/chosen": -0.3933795094490051,
      "logits/rejected": -0.4303474426269531,
      "logps/chosen": -9.630545616149902,
      "logps/rejected": -25.0108642578125,
      "loss": 1.1940371990203857,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6422770023345947,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.19723013043403625,
      "rewards/margins": 0.8150386214256287,
      "rewards/rejected": -1.0122687816619873,
      "step": 39,
      "train_speed(iter/s)": 0.011247
    },
    {
      "epoch": 0.10059729644765797,
      "grad_norm": 2.497746229171753,
      "learning_rate": 0.00013333333333333334,
      "logits/chosen": -0.3561173677444458,
      "logits/rejected": -0.4964173436164856,
      "logps/chosen": -6.288295269012451,
      "logps/rejected": -28.722339630126953,
      "loss": 0.7734813690185547,
      "memory(GiB)": 40.85,
      "nll_loss": 0.35979557037353516,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.020053118467330933,
      "rewards/margins": 1.0713183879852295,
      "rewards/rejected": -1.0512654781341553,
      "step": 40,
      "train_speed(iter/s)": 0.011248
    },
    {
      "epoch": 0.10311222885884942,
      "grad_norm": 3.900786876678467,
      "learning_rate": 0.00013666666666666666,
      "logits/chosen": -0.30438148975372314,
      "logits/rejected": -0.4660162031650543,
      "logps/chosen": -9.80126953125,
      "logps/rejected": -22.89883804321289,
      "loss": 1.0617499351501465,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5348615646362305,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.037312038242816925,
      "rewards/margins": 0.48836010694503784,
      "rewards/rejected": -0.4510480761528015,
      "step": 41,
      "train_speed(iter/s)": 0.011248
    },
    {
      "epoch": 0.10562716127004086,
      "grad_norm": 4.212996006011963,
      "learning_rate": 0.00014,
      "logits/chosen": -0.37392380833625793,
      "logits/rejected": -0.4811669588088989,
      "logps/chosen": -8.54138469696045,
      "logps/rejected": -24.88803482055664,
      "loss": 0.9449828267097473,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40984830260276794,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.078499935567379,
      "rewards/margins": 0.6149317026138306,
      "rewards/rejected": -0.6934316754341125,
      "step": 42,
      "train_speed(iter/s)": 0.011249
    },
    {
      "epoch": 0.10814209368123232,
      "grad_norm": 2.7297940254211426,
      "learning_rate": 0.00014333333333333334,
      "logits/chosen": -0.22235050797462463,
      "logits/rejected": -0.44656193256378174,
      "logps/chosen": -3.8628368377685547,
      "logps/rejected": -29.070890426635742,
      "loss": 0.6864454746246338,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3404179513454437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10101112723350525,
      "rewards/margins": 1.1784940958023071,
      "rewards/rejected": -1.0774829387664795,
      "step": 43,
      "train_speed(iter/s)": 0.011247
    },
    {
      "epoch": 0.11065702609242377,
      "grad_norm": 4.357929229736328,
      "learning_rate": 0.00014666666666666666,
      "logits/chosen": -0.3152369558811188,
      "logits/rejected": -0.4675106406211853,
      "logps/chosen": -8.47473430633545,
      "logps/rejected": -35.49666976928711,
      "loss": 0.8841890096664429,
      "memory(GiB)": 40.85,
      "nll_loss": 0.543793261051178,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2599925100803375,
      "rewards/margins": 1.5472975969314575,
      "rewards/rejected": -1.8072898387908936,
      "step": 44,
      "train_speed(iter/s)": 0.011248
    },
    {
      "epoch": 0.11317195850361522,
      "grad_norm": 4.053813934326172,
      "learning_rate": 0.00015000000000000001,
      "logits/chosen": -0.47932106256484985,
      "logits/rejected": -0.5357992053031921,
      "logps/chosen": -10.139131546020508,
      "logps/rejected": -29.30261993408203,
      "loss": 1.0090879201889038,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6002873182296753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2646394371986389,
      "rewards/margins": 1.3857942819595337,
      "rewards/rejected": -1.6504336595535278,
      "step": 45,
      "train_speed(iter/s)": 0.01125
    },
    {
      "epoch": 0.11568689091480666,
      "grad_norm": 3.4071342945098877,
      "learning_rate": 0.00015333333333333334,
      "logits/chosen": -0.35965389013290405,
      "logits/rejected": -0.4886631965637207,
      "logps/chosen": -6.492125988006592,
      "logps/rejected": -33.90357971191406,
      "loss": 0.9477238655090332,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5519906878471375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13393068313598633,
      "rewards/margins": 1.3498430252075195,
      "rewards/rejected": -1.4837737083435059,
      "step": 46,
      "train_speed(iter/s)": 0.011251
    },
    {
      "epoch": 0.11820182332599811,
      "grad_norm": 1.5676898956298828,
      "learning_rate": 0.00015666666666666666,
      "logits/chosen": -0.3339039087295532,
      "logits/rejected": -0.5405474305152893,
      "logps/chosen": -5.18754768371582,
      "logps/rejected": -34.61940383911133,
      "loss": 0.5877978205680847,
      "memory(GiB)": 40.85,
      "nll_loss": 0.26233971118927,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04858784377574921,
      "rewards/margins": 1.4838428497314453,
      "rewards/rejected": -1.4352549314498901,
      "step": 47,
      "train_speed(iter/s)": 0.011252
    },
    {
      "epoch": 0.12071675573718957,
      "grad_norm": 1.7600055932998657,
      "learning_rate": 0.00016,
      "logits/chosen": -0.38554954528808594,
      "logits/rejected": -0.5349220633506775,
      "logps/chosen": -5.35024356842041,
      "logps/rejected": -30.936561584472656,
      "loss": 0.7445188760757446,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3484676480293274,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.04428477957844734,
      "rewards/margins": 1.158116340637207,
      "rewards/rejected": -1.2024011611938477,
      "step": 48,
      "train_speed(iter/s)": 0.011253
    },
    {
      "epoch": 0.12323168814838101,
      "grad_norm": 1.8286348581314087,
      "learning_rate": 0.00016333333333333334,
      "logits/chosen": -0.3060781955718994,
      "logits/rejected": -0.5333737134933472,
      "logps/chosen": -4.790343761444092,
      "logps/rejected": -32.540409088134766,
      "loss": 0.6686277985572815,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3360018730163574,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0967080146074295,
      "rewards/margins": 1.285821795463562,
      "rewards/rejected": -1.1891138553619385,
      "step": 49,
      "train_speed(iter/s)": 0.011254
    },
    {
      "epoch": 0.12574662055957248,
      "grad_norm": 1.8983184099197388,
      "learning_rate": 0.0001666666666666667,
      "logits/chosen": -0.22582519054412842,
      "logits/rejected": -0.47035256028175354,
      "logps/chosen": -6.48522424697876,
      "logps/rejected": -32.73487091064453,
      "loss": 0.8076327443122864,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3926582336425781,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.01478707417845726,
      "rewards/margins": 1.167083501815796,
      "rewards/rejected": -1.1522964239120483,
      "step": 50,
      "train_speed(iter/s)": 0.011253
    },
    {
      "epoch": 0.12826155297076391,
      "grad_norm": 2.4192955493927,
      "learning_rate": 0.00017,
      "logits/chosen": -0.30057138204574585,
      "logits/rejected": -0.4678650498390198,
      "logps/chosen": -7.537961483001709,
      "logps/rejected": -34.26948547363281,
      "loss": 0.8188683390617371,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4251425266265869,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.051345158368349075,
      "rewards/margins": 1.1266952753067017,
      "rewards/rejected": -1.178040623664856,
      "step": 51,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.13077648538195535,
      "grad_norm": 2.0905075073242188,
      "learning_rate": 0.00017333333333333334,
      "logits/chosen": -0.2889178395271301,
      "logits/rejected": -0.49443909525871277,
      "logps/chosen": -6.827200412750244,
      "logps/rejected": -44.23398971557617,
      "loss": 0.7206944227218628,
      "memory(GiB)": 40.85,
      "nll_loss": 0.39006102085113525,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06844954192638397,
      "rewards/margins": 2.328122854232788,
      "rewards/rejected": -2.3965721130371094,
      "step": 52,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.13329141779314682,
      "grad_norm": 1.9464434385299683,
      "learning_rate": 0.00017666666666666666,
      "logits/chosen": -0.30524271726608276,
      "logits/rejected": -0.5259577631950378,
      "logps/chosen": -8.820595741271973,
      "logps/rejected": -34.23869705200195,
      "loss": 0.8924205899238586,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5174475312232971,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.007829657755792141,
      "rewards/margins": 1.6594951152801514,
      "rewards/rejected": -1.6673249006271362,
      "step": 53,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.13580635020433826,
      "grad_norm": 2.15484356880188,
      "learning_rate": 0.00018,
      "logits/chosen": -0.36469364166259766,
      "logits/rejected": -0.5003997087478638,
      "logps/chosen": -9.828875541687012,
      "logps/rejected": -35.949886322021484,
      "loss": 0.8116447925567627,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4780670404434204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04445749148726463,
      "rewards/margins": 1.7651525735855103,
      "rewards/rejected": -1.7206950187683105,
      "step": 54,
      "train_speed(iter/s)": 0.011212
    },
    {
      "epoch": 0.1383212826155297,
      "grad_norm": 5.288140773773193,
      "learning_rate": 0.00018333333333333334,
      "logits/chosen": -0.357359915971756,
      "logits/rejected": -0.5485042333602905,
      "logps/chosen": -4.562381744384766,
      "logps/rejected": -40.544979095458984,
      "loss": 0.8048219680786133,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4945175051689148,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.026466388255357742,
      "rewards/margins": 2.5180537700653076,
      "rewards/rejected": -2.544520378112793,
      "step": 55,
      "train_speed(iter/s)": 0.011214
    },
    {
      "epoch": 0.14083621502672117,
      "grad_norm": 4.063258647918701,
      "learning_rate": 0.0001866666666666667,
      "logits/chosen": -0.39062973856925964,
      "logits/rejected": -0.5040320158004761,
      "logps/chosen": -10.983360290527344,
      "logps/rejected": -33.32127380371094,
      "loss": 1.1587333679199219,
      "memory(GiB)": 40.85,
      "nll_loss": 0.7283708453178406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13727927207946777,
      "rewards/margins": 1.9422543048858643,
      "rewards/rejected": -2.079533576965332,
      "step": 56,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.1433511474379126,
      "grad_norm": 3.7057104110717773,
      "learning_rate": 0.00019,
      "logits/chosen": -0.34116944670677185,
      "logits/rejected": -0.5513606667518616,
      "logps/chosen": -5.59161376953125,
      "logps/rejected": -47.29106903076172,
      "loss": 0.8299980759620667,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5494524836540222,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13214518129825592,
      "rewards/margins": 2.937675952911377,
      "rewards/rejected": -3.069821357727051,
      "step": 57,
      "train_speed(iter/s)": 0.011217
    },
    {
      "epoch": 0.14586607984910405,
      "grad_norm": 4.7582502365112305,
      "learning_rate": 0.00019333333333333333,
      "logits/chosen": -0.4154261350631714,
      "logits/rejected": -0.5971646904945374,
      "logps/chosen": -3.059924364089966,
      "logps/rejected": -43.29597091674805,
      "loss": 0.5003064274787903,
      "memory(GiB)": 40.85,
      "nll_loss": 0.25123852491378784,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07654222846031189,
      "rewards/margins": 2.7813808917999268,
      "rewards/rejected": -2.704838752746582,
      "step": 58,
      "train_speed(iter/s)": 0.011219
    },
    {
      "epoch": 0.14838101226029551,
      "grad_norm": 2.1735236644744873,
      "learning_rate": 0.00019666666666666666,
      "logits/chosen": -0.4227810800075531,
      "logits/rejected": -0.47660815715789795,
      "logps/chosen": -8.593082427978516,
      "logps/rejected": -26.549867630004883,
      "loss": 1.0610322952270508,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5426512360572815,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10025864094495773,
      "rewards/margins": 1.2511481046676636,
      "rewards/rejected": -1.3514068126678467,
      "step": 59,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.15089594467148695,
      "grad_norm": 2.805598258972168,
      "learning_rate": 0.0002,
      "logits/chosen": -0.300743043422699,
      "logits/rejected": -0.47928643226623535,
      "logps/chosen": -6.593828201293945,
      "logps/rejected": -33.23126983642578,
      "loss": 0.7988956570625305,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40749770402908325,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.0006714202463626862,
      "rewards/margins": 1.6283259391784668,
      "rewards/rejected": -1.6276545524597168,
      "step": 60,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.1534108770826784,
      "grad_norm": 2.290149211883545,
      "learning_rate": 0.0001999996142159566,
      "logits/chosen": -0.3385659158229828,
      "logits/rejected": -0.4102441966533661,
      "logps/chosen": -8.33851146697998,
      "logps/rejected": -29.508546829223633,
      "loss": 0.7559148073196411,
      "memory(GiB)": 40.85,
      "nll_loss": 0.35449326038360596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0228387713432312,
      "rewards/margins": 1.5235671997070312,
      "rewards/rejected": -1.5464060306549072,
      "step": 61,
      "train_speed(iter/s)": 0.011222
    },
    {
      "epoch": 0.15592580949386986,
      "grad_norm": 1.7177425622940063,
      "learning_rate": 0.0001999984568668029,
      "logits/chosen": -0.32937324047088623,
      "logits/rejected": -0.4484848380088806,
      "logps/chosen": -3.493271827697754,
      "logps/rejected": -34.78036880493164,
      "loss": 0.6279686093330383,
      "memory(GiB)": 40.85,
      "nll_loss": 0.2725304365158081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11392105370759964,
      "rewards/margins": 1.8176403045654297,
      "rewards/rejected": -1.703719139099121,
      "step": 62,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.1584407419050613,
      "grad_norm": 1.8914873600006104,
      "learning_rate": 0.00019999652796146876,
      "logits/chosen": -0.29077184200286865,
      "logits/rejected": -0.48235204815864563,
      "logps/chosen": -7.319096565246582,
      "logps/rejected": -39.54444885253906,
      "loss": 0.7298395037651062,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40350908041000366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0820951834321022,
      "rewards/margins": 1.882514476776123,
      "rewards/rejected": -1.8004192113876343,
      "step": 63,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.16095567431625274,
      "grad_norm": 1.429303526878357,
      "learning_rate": 0.0001999938275148369,
      "logits/chosen": -0.3300917446613312,
      "logits/rejected": -0.45771050453186035,
      "logps/chosen": -3.8289737701416016,
      "logps/rejected": -39.278968811035156,
      "loss": 0.39868107438087463,
      "memory(GiB)": 40.85,
      "nll_loss": 0.18221822381019592,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07021164149045944,
      "rewards/margins": 2.249926805496216,
      "rewards/rejected": -2.320138454437256,
      "step": 64,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.1634706067274442,
      "grad_norm": 2.639005661010742,
      "learning_rate": 0.00019999035554774314,
      "logits/chosen": -0.36244961619377136,
      "logits/rejected": -0.5052404403686523,
      "logps/chosen": -5.230596542358398,
      "logps/rejected": -39.43718338012695,
      "loss": 0.7721049189567566,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4854365885257721,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1088234931230545,
      "rewards/margins": 2.1535189151763916,
      "rewards/rejected": -2.2623424530029297,
      "step": 65,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.16598553913863565,
      "grad_norm": 2.576199769973755,
      "learning_rate": 0.00019998611208697607,
      "logits/chosen": -0.3720204830169678,
      "logits/rejected": -0.507696270942688,
      "logps/chosen": -4.436941146850586,
      "logps/rejected": -34.991371154785156,
      "loss": 0.7472538352012634,
      "memory(GiB)": 40.85,
      "nll_loss": 0.39483439922332764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0759558230638504,
      "rewards/margins": 1.7400535345077515,
      "rewards/rejected": -1.816009521484375,
      "step": 66,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.16850047154982709,
      "grad_norm": 1.724285364151001,
      "learning_rate": 0.00019998109716527686,
      "logits/chosen": -0.25216248631477356,
      "logits/rejected": -0.4494931101799011,
      "logps/chosen": -3.201148748397827,
      "logps/rejected": -39.95793533325195,
      "loss": 0.4833192229270935,
      "memory(GiB)": 40.85,
      "nll_loss": 0.20408841967582703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06092430651187897,
      "rewards/margins": 2.2583165168762207,
      "rewards/rejected": -2.197392225265503,
      "step": 67,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.17101540396101855,
      "grad_norm": 2.91644287109375,
      "learning_rate": 0.00019997531082133904,
      "logits/chosen": -0.30811840295791626,
      "logits/rejected": -0.413838654756546,
      "logps/chosen": -7.5586838722229,
      "logps/rejected": -29.464954376220703,
      "loss": 1.0136064291000366,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5009399652481079,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0595913827419281,
      "rewards/margins": 1.2450876235961914,
      "rewards/rejected": -1.304679036140442,
      "step": 68,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.17353033637221,
      "grad_norm": 1.6453865766525269,
      "learning_rate": 0.00019996875309980826,
      "logits/chosen": -0.32626980543136597,
      "logits/rejected": -0.47710081934928894,
      "logps/chosen": -4.140695095062256,
      "logps/rejected": -33.830894470214844,
      "loss": 0.6451548933982849,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3717566728591919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.094700388610363,
      "rewards/margins": 1.8348017930984497,
      "rewards/rejected": -1.7401013374328613,
      "step": 69,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.17604526878340146,
      "grad_norm": 2.180673360824585,
      "learning_rate": 0.0001999614240512817,
      "logits/chosen": -0.2670721411705017,
      "logits/rejected": -0.46321332454681396,
      "logps/chosen": -7.102023601531982,
      "logps/rejected": -27.861366271972656,
      "loss": 0.7481832504272461,
      "memory(GiB)": 40.85,
      "nll_loss": 0.37576019763946533,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16741110384464264,
      "rewards/margins": 1.3596775531768799,
      "rewards/rejected": -1.1922664642333984,
      "step": 70,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.1785602011945929,
      "grad_norm": 2.049133539199829,
      "learning_rate": 0.00019995332373230808,
      "logits/chosen": -0.241715669631958,
      "logits/rejected": -0.407115638256073,
      "logps/chosen": -4.861550331115723,
      "logps/rejected": -34.9086799621582,
      "loss": 0.6162200570106506,
      "memory(GiB)": 40.85,
      "nll_loss": 0.25812438130378723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.037743180990219116,
      "rewards/margins": 1.569511890411377,
      "rewards/rejected": -1.5317686796188354,
      "step": 71,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.18107513360578434,
      "grad_norm": 4.052990913391113,
      "learning_rate": 0.00019994445220538677,
      "logits/chosen": -0.2541053295135498,
      "logits/rejected": -0.36276954412460327,
      "logps/chosen": -7.95726203918457,
      "logps/rejected": -36.47919845581055,
      "loss": 0.9209573268890381,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5536596179008484,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.17974892258644104,
      "rewards/margins": 1.929343581199646,
      "rewards/rejected": -2.1090924739837646,
      "step": 72,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.1835900660169758,
      "grad_norm": 3.210548162460327,
      "learning_rate": 0.0001999348095389677,
      "logits/chosen": -0.24167482554912567,
      "logits/rejected": -0.45636847615242004,
      "logps/chosen": -7.025866508483887,
      "logps/rejected": -48.629905700683594,
      "loss": 0.7358823418617249,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4621264636516571,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.05025668814778328,
      "rewards/margins": 2.2894766330718994,
      "rewards/rejected": -2.339733123779297,
      "step": 73,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.18610499842816725,
      "grad_norm": 4.565255165100098,
      "learning_rate": 0.0001999243958074506,
      "logits/chosen": -0.21860051155090332,
      "logits/rejected": -0.42088112235069275,
      "logps/chosen": -13.931788444519043,
      "logps/rejected": -42.80027770996094,
      "loss": 1.2019851207733154,
      "memory(GiB)": 40.85,
      "nll_loss": 0.7477802038192749,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.36434897780418396,
      "rewards/margins": 1.4156924486160278,
      "rewards/rejected": -1.7800413370132446,
      "step": 74,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.18861993083935868,
      "grad_norm": 2.291332244873047,
      "learning_rate": 0.00019991321109118448,
      "logits/chosen": -0.39398425817489624,
      "logits/rejected": -0.44725537300109863,
      "logps/chosen": -6.393754482269287,
      "logps/rejected": -27.47494888305664,
      "loss": 0.7488479018211365,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3586839437484741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.012238548137247562,
      "rewards/margins": 1.2588202953338623,
      "rewards/rejected": -1.246581792831421,
      "step": 75,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.19113486325055015,
      "grad_norm": 1.830254077911377,
      "learning_rate": 0.00019990125547646704,
      "logits/chosen": -0.31489449739456177,
      "logits/rejected": -0.47091639041900635,
      "logps/chosen": -2.980022668838501,
      "logps/rejected": -29.753929138183594,
      "loss": 0.6747189164161682,
      "memory(GiB)": 40.85,
      "nll_loss": 0.2754742503166199,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03740588575601578,
      "rewards/margins": 1.4201977252960205,
      "rewards/rejected": -1.3827917575836182,
      "step": 76,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.1936497956617416,
      "grad_norm": 1.903449296951294,
      "learning_rate": 0.00019988852905554404,
      "logits/chosen": -0.2765476703643799,
      "logits/rejected": -0.4408729076385498,
      "logps/chosen": -5.630273342132568,
      "logps/rejected": -31.533220291137695,
      "loss": 0.7030317783355713,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3307255208492279,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1330985128879547,
      "rewards/margins": 1.2641156911849976,
      "rewards/rejected": -1.1310172080993652,
      "step": 77,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.19616472807293303,
      "grad_norm": 1.736433744430542,
      "learning_rate": 0.0001998750319266084,
      "logits/chosen": -0.3374086618423462,
      "logits/rejected": -0.48207730054855347,
      "logps/chosen": -5.3590593338012695,
      "logps/rejected": -29.23048973083496,
      "loss": 0.8495813608169556,
      "memory(GiB)": 40.85,
      "nll_loss": 0.37136876583099365,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.09036409854888916,
      "rewards/margins": 1.2102724313735962,
      "rewards/rejected": -1.119908332824707,
      "step": 78,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.1986796604841245,
      "grad_norm": 2.0839672088623047,
      "learning_rate": 0.00019986076419379974,
      "logits/chosen": -0.26226145029067993,
      "logits/rejected": -0.4474099576473236,
      "logps/chosen": -7.3566060066223145,
      "logps/rejected": -40.514286041259766,
      "loss": 0.9187750220298767,
      "memory(GiB)": 40.85,
      "nll_loss": 0.60602867603302,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09458937495946884,
      "rewards/margins": 1.9139156341552734,
      "rewards/rejected": -1.8193262815475464,
      "step": 79,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.20119459289531594,
      "grad_norm": 2.656200885772705,
      "learning_rate": 0.00019984572596720324,
      "logits/chosen": -0.4218776226043701,
      "logits/rejected": -0.548494279384613,
      "logps/chosen": -5.5665717124938965,
      "logps/rejected": -34.339046478271484,
      "loss": 0.7904801964759827,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4805757403373718,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07708686590194702,
      "rewards/margins": 1.846845269203186,
      "rewards/rejected": -1.9239320755004883,
      "step": 80,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.20370952530650738,
      "grad_norm": 1.582141399383545,
      "learning_rate": 0.00019982991736284915,
      "logits/chosen": -0.384469211101532,
      "logits/rejected": -0.48424291610717773,
      "logps/chosen": -6.730848789215088,
      "logps/rejected": -35.332828521728516,
      "loss": 0.6736842393875122,
      "memory(GiB)": 40.85,
      "nll_loss": 0.38718143105506897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02753191813826561,
      "rewards/margins": 2.01827073097229,
      "rewards/rejected": -1.9907389879226685,
      "step": 81,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.20622445771769884,
      "grad_norm": 10.319942474365234,
      "learning_rate": 0.00019981333850271158,
      "logits/chosen": -0.3957505226135254,
      "logits/rejected": -0.5100387334823608,
      "logps/chosen": -7.3371148109436035,
      "logps/rejected": -40.714134216308594,
      "loss": 0.7473828196525574,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4370650351047516,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05588718503713608,
      "rewards/margins": 2.654069423675537,
      "rewards/rejected": -2.709956645965576,
      "step": 82,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.20873939012889028,
      "grad_norm": 2.3877530097961426,
      "learning_rate": 0.0001997959895147077,
      "logits/chosen": -0.36577096581459045,
      "logits/rejected": -0.5359358787536621,
      "logps/chosen": -4.45527458190918,
      "logps/rejected": -46.861942291259766,
      "loss": 0.5462505221366882,
      "memory(GiB)": 40.85,
      "nll_loss": 0.30273228883743286,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15123440325260162,
      "rewards/margins": 2.9262986183166504,
      "rewards/rejected": -2.77506422996521,
      "step": 83,
      "train_speed(iter/s)": 0.011235
    },
    {
      "epoch": 0.21125432254008172,
      "grad_norm": 2.1806390285491943,
      "learning_rate": 0.0001997778705326968,
      "logits/chosen": -0.3485352396965027,
      "logits/rejected": -0.5878627300262451,
      "logps/chosen": -2.89451265335083,
      "logps/rejected": -44.27880859375,
      "loss": 0.6081856489181519,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3298896849155426,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05770353972911835,
      "rewards/margins": 2.5825228691101074,
      "rewards/rejected": -2.5248193740844727,
      "step": 84,
      "train_speed(iter/s)": 0.011235
    },
    {
      "epoch": 0.2137692549512732,
      "grad_norm": 2.80009388923645,
      "learning_rate": 0.00019975898169647915,
      "logits/chosen": -0.39257609844207764,
      "logits/rejected": -0.5535434484481812,
      "logps/chosen": -7.839548587799072,
      "logps/rejected": -42.132286071777344,
      "loss": 0.7175056338310242,
      "memory(GiB)": 40.85,
      "nll_loss": 0.405160129070282,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.025028400123119354,
      "rewards/margins": 2.475644111633301,
      "rewards/rejected": -2.450615644454956,
      "step": 85,
      "train_speed(iter/s)": 0.011235
    },
    {
      "epoch": 0.21628418736246463,
      "grad_norm": 3.3921735286712646,
      "learning_rate": 0.000199739323151795,
      "logits/chosen": -0.35620519518852234,
      "logits/rejected": -0.6006304025650024,
      "logps/chosen": -5.945646286010742,
      "logps/rejected": -45.728271484375,
      "loss": 0.6699403524398804,
      "memory(GiB)": 40.85,
      "nll_loss": 0.33112284541130066,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.012618173845112324,
      "rewards/margins": 2.700666666030884,
      "rewards/rejected": -2.713285207748413,
      "step": 86,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.21879911977365607,
      "grad_norm": 3.2411773204803467,
      "learning_rate": 0.00019971889505032334,
      "logits/chosen": -0.42666560411453247,
      "logits/rejected": -0.6167599558830261,
      "logps/chosen": -5.318458080291748,
      "logps/rejected": -49.244285583496094,
      "loss": 0.6356828808784485,
      "memory(GiB)": 40.85,
      "nll_loss": 0.39307594299316406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1333954930305481,
      "rewards/margins": 3.1458327770233154,
      "rewards/rejected": -3.2792282104492188,
      "step": 87,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.22131405218484754,
      "grad_norm": 1.924742341041565,
      "learning_rate": 0.00019969769754968098,
      "logits/chosen": -0.3413890600204468,
      "logits/rejected": -0.5464233160018921,
      "logps/chosen": -5.729033946990967,
      "logps/rejected": -41.974422454833984,
      "loss": 0.5986770987510681,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3535672724246979,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14028939604759216,
      "rewards/margins": 2.51139235496521,
      "rewards/rejected": -2.371102809906006,
      "step": 88,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.22382898459603898,
      "grad_norm": 2.161083459854126,
      "learning_rate": 0.00019967573081342103,
      "logits/chosen": -0.38623473048210144,
      "logits/rejected": -0.5679455399513245,
      "logps/chosen": -6.484611511230469,
      "logps/rejected": -38.930511474609375,
      "loss": 0.7250155806541443,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4377710521221161,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.024773895740509033,
      "rewards/margins": 2.3375205993652344,
      "rewards/rejected": -2.3622944355010986,
      "step": 89,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.22634391700723044,
      "grad_norm": 6.164645195007324,
      "learning_rate": 0.00019965299501103177,
      "logits/chosen": -0.33336132764816284,
      "logits/rejected": -0.5168649554252625,
      "logps/chosen": -11.845653533935547,
      "logps/rejected": -55.690311431884766,
      "loss": 1.0204803943634033,
      "memory(GiB)": 40.85,
      "nll_loss": 0.7011927962303162,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.22887784242630005,
      "rewards/margins": 3.111534357070923,
      "rewards/rejected": -3.3404123783111572,
      "step": 90,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.22885884941842188,
      "grad_norm": 2.4981861114501953,
      "learning_rate": 0.00019962949031793542,
      "logits/chosen": -0.4224125146865845,
      "logits/rejected": -0.5427595376968384,
      "logps/chosen": -5.719327926635742,
      "logps/rejected": -33.303741455078125,
      "loss": 0.7469587326049805,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4206176996231079,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.008410206064581871,
      "rewards/margins": 1.994816541671753,
      "rewards/rejected": -2.0032269954681396,
      "step": 91,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.23137378182961332,
      "grad_norm": 2.5139830112457275,
      "learning_rate": 0.00019960521691548674,
      "logits/chosen": -0.34622836112976074,
      "logits/rejected": -0.5427547097206116,
      "logps/chosen": -5.069763660430908,
      "logps/rejected": -41.81755828857422,
      "loss": 0.6454588770866394,
      "memory(GiB)": 40.85,
      "nll_loss": 0.35349082946777344,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.046083107590675354,
      "rewards/margins": 2.2620620727539062,
      "rewards/rejected": -2.30814528465271,
      "step": 92,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.2338887142408048,
      "grad_norm": 2.1890244483947754,
      "learning_rate": 0.0001995801749909715,
      "logits/chosen": -0.3081744611263275,
      "logits/rejected": -0.47990015149116516,
      "logps/chosen": -6.5690107345581055,
      "logps/rejected": -34.549991607666016,
      "loss": 0.6765730381011963,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3245474696159363,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.060123831033706665,
      "rewards/margins": 1.8496551513671875,
      "rewards/rejected": -1.7895313501358032,
      "step": 93,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.23640364665199623,
      "grad_norm": 1.8172200918197632,
      "learning_rate": 0.00019955436473760525,
      "logits/chosen": -0.24766620993614197,
      "logits/rejected": -0.49844813346862793,
      "logps/chosen": -5.177490711212158,
      "logps/rejected": -48.42265701293945,
      "loss": 0.7104548811912537,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4418777823448181,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10037174820899963,
      "rewards/margins": 2.629190683364868,
      "rewards/rejected": -2.5288188457489014,
      "step": 94,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.23891857906318767,
      "grad_norm": 2.0303428173065186,
      "learning_rate": 0.0001995277863545316,
      "logits/chosen": -0.3552500307559967,
      "logits/rejected": -0.5088093876838684,
      "logps/chosen": -3.972975730895996,
      "logps/rejected": -29.582290649414062,
      "loss": 0.5607516169548035,
      "memory(GiB)": 42.61,
      "nll_loss": 0.27718478441238403,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09949840605258942,
      "rewards/margins": 1.6923424005508423,
      "rewards/rejected": -1.592844009399414,
      "step": 95,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.24143351147437914,
      "grad_norm": 2.3172338008880615,
      "learning_rate": 0.00019950044004682092,
      "logits/chosen": -0.30597299337387085,
      "logits/rejected": -0.4608762860298157,
      "logps/chosen": -10.467157363891602,
      "logps/rejected": -44.41074752807617,
      "loss": 0.9089665412902832,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5540332794189453,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0023995572701096535,
      "rewards/margins": 2.6161789894104004,
      "rewards/rejected": -2.6185781955718994,
      "step": 96,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.24394844388557058,
      "grad_norm": 1.6338391304016113,
      "learning_rate": 0.0001994723260254686,
      "logits/chosen": -0.20743098855018616,
      "logits/rejected": -0.4207964837551117,
      "logps/chosen": -4.458022117614746,
      "logps/rejected": -48.164268493652344,
      "loss": 0.5020785927772522,
      "memory(GiB)": 42.61,
      "nll_loss": 0.2824496626853943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15383610129356384,
      "rewards/margins": 3.0794906616210938,
      "rewards/rejected": -2.925654172897339,
      "step": 97,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.24646337629676202,
      "grad_norm": 2.5537126064300537,
      "learning_rate": 0.0001994434445073934,
      "logits/chosen": -0.1935003101825714,
      "logits/rejected": -0.4715961217880249,
      "logps/chosen": -3.131989002227783,
      "logps/rejected": -61.07079315185547,
      "loss": 0.3960198163986206,
      "memory(GiB)": 42.61,
      "nll_loss": 0.21717773377895355,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07503610849380493,
      "rewards/margins": 4.007692337036133,
      "rewards/rejected": -3.9326562881469727,
      "step": 98,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.24897830870795348,
      "grad_norm": 7.057893753051758,
      "learning_rate": 0.00019941379571543596,
      "logits/chosen": -0.21255502104759216,
      "logits/rejected": -0.4117867946624756,
      "logps/chosen": -5.764989852905273,
      "logps/rejected": -59.11027908325195,
      "loss": 0.7163435816764832,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4923526346683502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1150842159986496,
      "rewards/margins": 3.945159912109375,
      "rewards/rejected": -4.060243606567383,
      "step": 99,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.25149324111914495,
      "grad_norm": 6.774058818817139,
      "learning_rate": 0.00019938337987835688,
      "logits/chosen": -0.2647398114204407,
      "logits/rejected": -0.34906458854675293,
      "logps/chosen": -10.037410736083984,
      "logps/rejected": -37.84819030761719,
      "loss": 1.0176275968551636,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5385105609893799,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.28268206119537354,
      "rewards/margins": 2.0650789737701416,
      "rewards/rejected": -2.3477611541748047,
      "step": 100,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.2540081735303364,
      "grad_norm": 1.4746135473251343,
      "learning_rate": 0.00019935219723083502,
      "logits/chosen": -0.28873857855796814,
      "logits/rejected": -0.4278784990310669,
      "logps/chosen": -4.487892150878906,
      "logps/rejected": -43.24613952636719,
      "loss": 0.5632098317146301,
      "memory(GiB)": 42.61,
      "nll_loss": 0.31394267082214355,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09316689521074295,
      "rewards/margins": 3.0039639472961426,
      "rewards/rejected": -2.910796880722046,
      "step": 101,
      "train_speed(iter/s)": 0.011219
    },
    {
      "epoch": 0.25652310594152783,
      "grad_norm": 2.2414140701293945,
      "learning_rate": 0.0001993202480134658,
      "logits/chosen": -0.286653995513916,
      "logits/rejected": -0.4876902103424072,
      "logps/chosen": -2.5823984146118164,
      "logps/rejected": -50.10431671142578,
      "loss": 0.3667687475681305,
      "memory(GiB)": 42.61,
      "nll_loss": 0.19701245427131653,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11824626475572586,
      "rewards/margins": 3.1774046421051025,
      "rewards/rejected": -3.0591585636138916,
      "step": 102,
      "train_speed(iter/s)": 0.011219
    },
    {
      "epoch": 0.25903803835271927,
      "grad_norm": 2.5298242568969727,
      "learning_rate": 0.0001992875324727591,
      "logits/chosen": -0.2699410319328308,
      "logits/rejected": -0.421194463968277,
      "logps/chosen": -7.033985137939453,
      "logps/rejected": -51.89061737060547,
      "loss": 0.6059828400611877,
      "memory(GiB)": 42.61,
      "nll_loss": 0.37160247564315796,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07941240072250366,
      "rewards/margins": 3.3084330558776855,
      "rewards/rejected": -3.387845039367676,
      "step": 103,
      "train_speed(iter/s)": 0.01122
    },
    {
      "epoch": 0.2615529707639107,
      "grad_norm": 5.99639368057251,
      "learning_rate": 0.00019925405086113768,
      "logits/chosen": -0.23520317673683167,
      "logits/rejected": -0.3863908052444458,
      "logps/chosen": -6.1102681159973145,
      "logps/rejected": -41.600799560546875,
      "loss": 0.8184252381324768,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5032728910446167,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.09978590905666351,
      "rewards/margins": 2.611102819442749,
      "rewards/rejected": -2.7108888626098633,
      "step": 104,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.26406790317510215,
      "grad_norm": 4.219518184661865,
      "learning_rate": 0.0001992198034369349,
      "logits/chosen": -0.2237715721130371,
      "logits/rejected": -0.38729795813560486,
      "logps/chosen": -6.913872718811035,
      "logps/rejected": -42.98509979248047,
      "loss": 0.769071638584137,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5157548189163208,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.060354869812726974,
      "rewards/margins": 2.590625762939453,
      "rewards/rejected": -2.530271053314209,
      "step": 105,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.26658283558629364,
      "grad_norm": 4.175528049468994,
      "learning_rate": 0.000199184790464393,
      "logits/chosen": -0.2919754683971405,
      "logits/rejected": -0.4413781762123108,
      "logps/chosen": -4.1272172927856445,
      "logps/rejected": -32.008277893066406,
      "loss": 0.7491403222084045,
      "memory(GiB)": 42.61,
      "nll_loss": 0.41593173146247864,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08565365523099899,
      "rewards/margins": 2.0251622200012207,
      "rewards/rejected": -1.9395084381103516,
      "step": 106,
      "train_speed(iter/s)": 0.011222
    },
    {
      "epoch": 0.2690977679974851,
      "grad_norm": 2.857898473739624,
      "learning_rate": 0.00019914901221366086,
      "logits/chosen": -0.2756834924221039,
      "logits/rejected": -0.4335779845714569,
      "logps/chosen": -5.15769624710083,
      "logps/rejected": -41.91234588623047,
      "loss": 0.63164883852005,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3513546586036682,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06739385426044464,
      "rewards/margins": 2.5824873447418213,
      "rewards/rejected": -2.5150933265686035,
      "step": 107,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.2716127004086765,
      "grad_norm": 1.9767286777496338,
      "learning_rate": 0.0001991124689607921,
      "logits/chosen": -0.23816093802452087,
      "logits/rejected": -0.4088671803474426,
      "logps/chosen": -6.058350563049316,
      "logps/rejected": -36.48659133911133,
      "loss": 0.795885443687439,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4665300250053406,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.02518155239522457,
      "rewards/margins": 2.0897223949432373,
      "rewards/rejected": -2.0645408630371094,
      "step": 108,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.27412763281986796,
      "grad_norm": 4.606684684753418,
      "learning_rate": 0.00019907516098774275,
      "logits/chosen": -0.3413383960723877,
      "logits/rejected": -0.4257667064666748,
      "logps/chosen": -5.4365692138671875,
      "logps/rejected": -36.4229621887207,
      "loss": 0.8568770885467529,
      "memory(GiB)": 42.61,
      "nll_loss": 0.528153121471405,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.014822255820035934,
      "rewards/margins": 2.660560369491577,
      "rewards/rejected": -2.645737886428833,
      "step": 109,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.2766425652310594,
      "grad_norm": 2.080709457397461,
      "learning_rate": 0.00019903708858236925,
      "logits/chosen": -0.22644048929214478,
      "logits/rejected": -0.3494427800178528,
      "logps/chosen": -6.887048721313477,
      "logps/rejected": -39.19084167480469,
      "loss": 0.7752804756164551,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4188353717327118,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.046474143862724304,
      "rewards/margins": 2.0502114295959473,
      "rewards/rejected": -2.0966856479644775,
      "step": 110,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.27915749764225084,
      "grad_norm": 2.133538007736206,
      "learning_rate": 0.00019899825203842613,
      "logits/chosen": -0.24960362911224365,
      "logits/rejected": -0.42648327350616455,
      "logps/chosen": -4.220504283905029,
      "logps/rejected": -38.22932815551758,
      "loss": 0.7358196377754211,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4154231548309326,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12325219810009003,
      "rewards/margins": 2.0720622539520264,
      "rewards/rejected": -1.9488099813461304,
      "step": 111,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.28167243005344234,
      "grad_norm": 1.9261109828948975,
      "learning_rate": 0.00019895865165556377,
      "logits/chosen": -0.2359025776386261,
      "logits/rejected": -0.4521249830722809,
      "logps/chosen": -5.312178611755371,
      "logps/rejected": -42.527828216552734,
      "loss": 0.6443281769752502,
      "memory(GiB)": 42.61,
      "nll_loss": 0.33275920152664185,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.00696786493062973,
      "rewards/margins": 2.2711665630340576,
      "rewards/rejected": -2.2781341075897217,
      "step": 112,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.2841873624646338,
      "grad_norm": 1.973440170288086,
      "learning_rate": 0.00019891828773932604,
      "logits/chosen": -0.31173762679100037,
      "logits/rejected": -0.4782518148422241,
      "logps/chosen": -5.571796417236328,
      "logps/rejected": -36.36091995239258,
      "loss": 0.6021603345870972,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3728388547897339,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.25308576226234436,
      "rewards/margins": 2.2811777591705322,
      "rewards/rejected": -2.0280919075012207,
      "step": 113,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.2867022948758252,
      "grad_norm": 2.1112029552459717,
      "learning_rate": 0.0001988771606011481,
      "logits/chosen": -0.29321083426475525,
      "logits/rejected": -0.4459826946258545,
      "logps/chosen": -7.568819046020508,
      "logps/rejected": -36.86845016479492,
      "loss": 0.8353914022445679,
      "memory(GiB)": 42.61,
      "nll_loss": 0.557527482509613,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08000842481851578,
      "rewards/margins": 2.1105971336364746,
      "rewards/rejected": -2.0305888652801514,
      "step": 114,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.28921722728701665,
      "grad_norm": 1.5973310470581055,
      "learning_rate": 0.0001988352705583538,
      "logits/chosen": -0.35277947783470154,
      "logits/rejected": -0.4771711826324463,
      "logps/chosen": -6.40337610244751,
      "logps/rejected": -36.56932830810547,
      "loss": 0.7110580801963806,
      "memory(GiB)": 42.61,
      "nll_loss": 0.380901962518692,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0295229684561491,
      "rewards/margins": 2.1401078701019287,
      "rewards/rejected": -2.1105847358703613,
      "step": 115,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.2917321596982081,
      "grad_norm": 2.2071542739868164,
      "learning_rate": 0.0001987926179341533,
      "logits/chosen": -0.3233400583267212,
      "logits/rejected": -0.49479973316192627,
      "logps/chosen": -7.279633522033691,
      "logps/rejected": -42.67523956298828,
      "loss": 0.6601336002349854,
      "memory(GiB)": 42.61,
      "nll_loss": 0.42307049036026,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14696629345417023,
      "rewards/margins": 2.9207940101623535,
      "rewards/rejected": -2.77382755279541,
      "step": 116,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.29424709210939953,
      "grad_norm": 7.165996551513672,
      "learning_rate": 0.00019874920305764068,
      "logits/chosen": -0.34290945529937744,
      "logits/rejected": -0.5152956247329712,
      "logps/chosen": -7.311550140380859,
      "logps/rejected": -41.441612243652344,
      "loss": 0.9759319424629211,
      "memory(GiB)": 42.61,
      "nll_loss": 0.6613105535507202,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.17784443497657776,
      "rewards/margins": 2.404524803161621,
      "rewards/rejected": -2.582369565963745,
      "step": 117,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.29676202452059103,
      "grad_norm": 2.9480412006378174,
      "learning_rate": 0.00019870502626379127,
      "logits/chosen": -0.3888930678367615,
      "logits/rejected": -0.5601359009742737,
      "logps/chosen": -6.752213954925537,
      "logps/rejected": -42.02001190185547,
      "loss": 0.6632174849510193,
      "memory(GiB)": 42.61,
      "nll_loss": 0.41683804988861084,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10449926555156708,
      "rewards/margins": 2.6166954040527344,
      "rewards/rejected": -2.721194267272949,
      "step": 118,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.29927695693178247,
      "grad_norm": 8.507670402526855,
      "learning_rate": 0.00019866008789345917,
      "logits/chosen": -0.40375155210494995,
      "logits/rejected": -0.5714547634124756,
      "logps/chosen": -6.916952610015869,
      "logps/rejected": -41.485107421875,
      "loss": 0.8475279211997986,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5039829015731812,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.07569311559200287,
      "rewards/margins": 2.307457685470581,
      "rewards/rejected": -2.383150577545166,
      "step": 119,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.3017918893429739,
      "grad_norm": 1.6022049188613892,
      "learning_rate": 0.00019861438829337438,
      "logits/chosen": -0.3536994159221649,
      "logits/rejected": -0.5457285046577454,
      "logps/chosen": -7.962162971496582,
      "logps/rejected": -50.184635162353516,
      "loss": 0.7923818826675415,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5494890809059143,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.098821260035038,
      "rewards/margins": 2.853030204772949,
      "rewards/rejected": -2.75420880317688,
      "step": 120,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.30430682175416535,
      "grad_norm": 4.437849521636963,
      "learning_rate": 0.00019856792781614054,
      "logits/chosen": -0.4381689429283142,
      "logits/rejected": -0.6431064605712891,
      "logps/chosen": -3.382079601287842,
      "logps/rejected": -36.976806640625,
      "loss": 0.7761183977127075,
      "memory(GiB)": 42.61,
      "nll_loss": 0.43555858731269836,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.0065191928297281265,
      "rewards/margins": 1.8534878492355347,
      "rewards/rejected": -1.846968650817871,
      "step": 121,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.3068217541653568,
      "grad_norm": 2.79468035697937,
      "learning_rate": 0.00019852070682023176,
      "logits/chosen": -0.5022692680358887,
      "logits/rejected": -0.6749197244644165,
      "logps/chosen": -6.771264553070068,
      "logps/rejected": -38.43341827392578,
      "loss": 0.6576388478279114,
      "memory(GiB)": 42.61,
      "nll_loss": 0.40165674686431885,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21301744878292084,
      "rewards/margins": 2.2969624996185303,
      "rewards/rejected": -2.0839452743530273,
      "step": 122,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.3093366865765483,
      "grad_norm": 1.964633822441101,
      "learning_rate": 0.00019847272566999026,
      "logits/chosen": -0.5233901739120483,
      "logits/rejected": -0.6417133212089539,
      "logps/chosen": -9.739703178405762,
      "logps/rejected": -28.968997955322266,
      "loss": 0.8914016485214233,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5449220538139343,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16006287932395935,
      "rewards/margins": 1.5245118141174316,
      "rewards/rejected": -1.3644486665725708,
      "step": 123,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.3118516189877397,
      "grad_norm": 1.871849775314331,
      "learning_rate": 0.0001984239847356233,
      "logits/chosen": -0.5555144548416138,
      "logits/rejected": -0.6761674284934998,
      "logps/chosen": -4.62461519241333,
      "logps/rejected": -41.918800354003906,
      "loss": 0.5709429383277893,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3194217383861542,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10139736533164978,
      "rewards/margins": 3.0129401683807373,
      "rewards/rejected": -2.9115428924560547,
      "step": 124,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.31436655139893116,
      "grad_norm": 2.3828136920928955,
      "learning_rate": 0.00019837448439320027,
      "logits/chosen": -0.5682411789894104,
      "logits/rejected": -0.7420387268066406,
      "logps/chosen": -3.964232921600342,
      "logps/rejected": -61.1876335144043,
      "loss": 0.5395267605781555,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3251190483570099,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.031132016330957413,
      "rewards/margins": 4.458404541015625,
      "rewards/rejected": -4.427271842956543,
      "step": 125,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.3168814838101226,
      "grad_norm": 2.152522325515747,
      "learning_rate": 0.0001983242250246501,
      "logits/chosen": -0.5287045240402222,
      "logits/rejected": -0.7190724611282349,
      "logps/chosen": -6.970052719116211,
      "logps/rejected": -73.82839965820312,
      "loss": 0.5324504971504211,
      "memory(GiB)": 42.61,
      "nll_loss": 0.40117147564888,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.012534398585557938,
      "rewards/margins": 5.6087493896484375,
      "rewards/rejected": -5.621283054351807,
      "step": 126,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.31939641622131404,
      "grad_norm": 5.082659721374512,
      "learning_rate": 0.00019827320701775806,
      "logits/chosen": -0.4820547103881836,
      "logits/rejected": -0.6797658205032349,
      "logps/chosen": -6.213442802429199,
      "logps/rejected": -70.91389465332031,
      "loss": 0.6097095012664795,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4043075144290924,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03578203544020653,
      "rewards/margins": 4.8603901863098145,
      "rewards/rejected": -4.896172046661377,
      "step": 127,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.3219113486325055,
      "grad_norm": 1.7538974285125732,
      "learning_rate": 0.0001982214307661627,
      "logits/chosen": -0.45252111554145813,
      "logits/rejected": -0.6814027428627014,
      "logps/chosen": -4.733944416046143,
      "logps/rejected": -62.52580261230469,
      "loss": 0.5949385166168213,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3638124167919159,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1602221429347992,
      "rewards/margins": 4.626731872558594,
      "rewards/rejected": -4.466509819030762,
      "step": 128,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.324426281043697,
      "grad_norm": 1.4243698120117188,
      "learning_rate": 0.00019816889666935317,
      "logits/chosen": -0.3787720203399658,
      "logits/rejected": -0.6261472702026367,
      "logps/chosen": -6.471246719360352,
      "logps/rejected": -71.88873291015625,
      "loss": 0.49682942032814026,
      "memory(GiB)": 42.61,
      "nll_loss": 0.33228519558906555,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13370710611343384,
      "rewards/margins": 4.678413391113281,
      "rewards/rejected": -4.544705867767334,
      "step": 129,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.3269412134548884,
      "grad_norm": 2.5607693195343018,
      "learning_rate": 0.00019811560513266572,
      "logits/chosen": -0.4134800434112549,
      "logits/rejected": -0.611393928527832,
      "logps/chosen": -5.9260759353637695,
      "logps/rejected": -60.01020431518555,
      "loss": 0.601435124874115,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4060709476470947,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08814961463212967,
      "rewards/margins": 4.3574066162109375,
      "rewards/rejected": -4.269256591796875,
      "step": 130,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.32945614586607985,
      "grad_norm": 1.9927021265029907,
      "learning_rate": 0.00019806155656728084,
      "logits/chosen": -0.33065900206565857,
      "logits/rejected": -0.6154841184616089,
      "logps/chosen": -7.140951156616211,
      "logps/rejected": -73.75382232666016,
      "loss": 0.7812880277633667,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5943235158920288,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07201401889324188,
      "rewards/margins": 4.931735992431641,
      "rewards/rejected": -5.003749370574951,
      "step": 131,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.3319710782772713,
      "grad_norm": 2.2726259231567383,
      "learning_rate": 0.00019800675139022006,
      "logits/chosen": -0.48140621185302734,
      "logits/rejected": -0.5989112257957458,
      "logps/chosen": -8.895767211914062,
      "logps/rejected": -55.02824783325195,
      "loss": 0.6833999156951904,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3953901529312134,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.2496323436498642,
      "rewards/margins": 3.696012258529663,
      "rewards/rejected": -3.9456448554992676,
      "step": 132,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.33448601068846273,
      "grad_norm": 2.0731728076934814,
      "learning_rate": 0.00019795119002434262,
      "logits/chosen": -0.4898430407047272,
      "logits/rejected": -0.5940038561820984,
      "logps/chosen": -6.4923415184021,
      "logps/rejected": -53.08872985839844,
      "loss": 0.6633037328720093,
      "memory(GiB)": 42.61,
      "nll_loss": 0.38765111565589905,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.014707805588841438,
      "rewards/margins": 3.6382081508636475,
      "rewards/rejected": -3.623500108718872,
      "step": 133,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.33700094309965417,
      "grad_norm": 2.124537467956543,
      "learning_rate": 0.00019789487289834228,
      "logits/chosen": -0.38429343700408936,
      "logits/rejected": -0.587513267993927,
      "logps/chosen": -3.70666241645813,
      "logps/rejected": -56.821434020996094,
      "loss": 0.5152694582939148,
      "memory(GiB)": 42.61,
      "nll_loss": 0.30610963702201843,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09033096581697464,
      "rewards/margins": 4.2761945724487305,
      "rewards/rejected": -4.185863494873047,
      "step": 134,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.33951587551084567,
      "grad_norm": 1.165412425994873,
      "learning_rate": 0.000197837800446744,
      "logits/chosen": -0.2946363687515259,
      "logits/rejected": -0.6158965229988098,
      "logps/chosen": -5.17818021774292,
      "logps/rejected": -85.47969818115234,
      "loss": 0.34627044200897217,
      "memory(GiB)": 42.61,
      "nll_loss": 0.2048141062259674,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.022251788526773453,
      "rewards/margins": 6.3592729568481445,
      "rewards/rejected": -6.337022304534912,
      "step": 135,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.3420308079220371,
      "grad_norm": 1.6704083681106567,
      "learning_rate": 0.00019777997310990063,
      "logits/chosen": -0.34255802631378174,
      "logits/rejected": -0.6556228995323181,
      "logps/chosen": -2.7305595874786377,
      "logps/rejected": -83.5248031616211,
      "loss": 0.355721652507782,
      "memory(GiB)": 42.61,
      "nll_loss": 0.25845229625701904,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13533437252044678,
      "rewards/margins": 6.411257266998291,
      "rewards/rejected": -6.275922775268555,
      "step": 136,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.34454574033322855,
      "grad_norm": 1.8042700290679932,
      "learning_rate": 0.00019772139133398942,
      "logits/chosen": -0.3927345275878906,
      "logits/rejected": -0.6234123706817627,
      "logps/chosen": -5.4586615562438965,
      "logps/rejected": -81.85598754882812,
      "loss": 0.4663994014263153,
      "memory(GiB)": 42.61,
      "nll_loss": 0.2979077696800232,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.012417782098054886,
      "rewards/margins": 6.194499492645264,
      "rewards/rejected": -6.2069172859191895,
      "step": 137,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.34706067274442,
      "grad_norm": 2.92842960357666,
      "learning_rate": 0.00019766205557100868,
      "logits/chosen": -0.559550940990448,
      "logits/rejected": -0.7328889966011047,
      "logps/chosen": -9.122381210327148,
      "logps/rejected": -70.6622314453125,
      "loss": 0.7152190208435059,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5275388956069946,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.06365346908569336,
      "rewards/margins": 5.660825729370117,
      "rewards/rejected": -5.724478721618652,
      "step": 138,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.3495756051556114,
      "grad_norm": 1.7731250524520874,
      "learning_rate": 0.00019760196627877422,
      "logits/chosen": -0.3356359004974365,
      "logits/rejected": -0.7464644908905029,
      "logps/chosen": -3.0185112953186035,
      "logps/rejected": -116.795654296875,
      "loss": 0.4166179299354553,
      "memory(GiB)": 42.61,
      "nll_loss": 0.28380244970321655,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07928900420665741,
      "rewards/margins": 9.400187492370605,
      "rewards/rejected": -9.32089900970459,
      "step": 139,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.3520905375668029,
      "grad_norm": 6.910381317138672,
      "learning_rate": 0.0001975411239209158,
      "logits/chosen": -0.6025805473327637,
      "logits/rejected": -0.7230421900749207,
      "logps/chosen": -13.585058212280273,
      "logps/rejected": -62.04022979736328,
      "loss": 1.3271528482437134,
      "memory(GiB)": 42.61,
      "nll_loss": 0.8942356109619141,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4771670699119568,
      "rewards/margins": 4.614741802215576,
      "rewards/rejected": -5.0919084548950195,
      "step": 140,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.35460546997799436,
      "grad_norm": 5.268382549285889,
      "learning_rate": 0.0001974795289668737,
      "logits/chosen": -0.47753405570983887,
      "logits/rejected": -0.8787558674812317,
      "logps/chosen": -8.216950416564941,
      "logps/rejected": -102.49224090576172,
      "loss": 0.7080188393592834,
      "memory(GiB)": 42.61,
      "nll_loss": 0.6081770658493042,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.37120816111564636,
      "rewards/margins": 7.621390342712402,
      "rewards/rejected": -7.992597579956055,
      "step": 141,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.3571204023891858,
      "grad_norm": 6.239411354064941,
      "learning_rate": 0.00019741718189189485,
      "logits/chosen": -0.5375499725341797,
      "logits/rejected": -0.7797288298606873,
      "logps/chosen": -7.561271667480469,
      "logps/rejected": -72.9504623413086,
      "loss": 0.9173229932785034,
      "memory(GiB)": 42.61,
      "nll_loss": 0.6234984397888184,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.2173769325017929,
      "rewards/margins": 5.198176383972168,
      "rewards/rejected": -5.415553092956543,
      "step": 142,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.35963533480037724,
      "grad_norm": 4.152512073516846,
      "learning_rate": 0.00019735408317702948,
      "logits/chosen": -0.5347225069999695,
      "logits/rejected": -0.7430693507194519,
      "logps/chosen": -4.227977752685547,
      "logps/rejected": -57.00012969970703,
      "loss": 0.5255740284919739,
      "memory(GiB)": 42.61,
      "nll_loss": 0.34119245409965515,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0003469698131084442,
      "rewards/margins": 4.147022247314453,
      "rewards/rejected": -4.1466755867004395,
      "step": 143,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.3621502672115687,
      "grad_norm": 1.935512900352478,
      "learning_rate": 0.0001972902333091271,
      "logits/chosen": -0.4890541732311249,
      "logits/rejected": -0.6482999920845032,
      "logps/chosen": -8.091506958007812,
      "logps/rejected": -47.94240951538086,
      "loss": 0.6421104669570923,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4184344708919525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.008477868512272835,
      "rewards/margins": 3.3832898139953613,
      "rewards/rejected": -3.3748116493225098,
      "step": 144,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.3646651996227601,
      "grad_norm": 3.257821559906006,
      "learning_rate": 0.00019722563278083287,
      "logits/chosen": -0.46462729573249817,
      "logits/rejected": -0.71808260679245,
      "logps/chosen": -3.831825017929077,
      "logps/rejected": -43.67421340942383,
      "loss": 0.6770502924919128,
      "memory(GiB)": 42.61,
      "nll_loss": 0.40698307752609253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03249932453036308,
      "rewards/margins": 2.939378023147583,
      "rewards/rejected": -2.9068784713745117,
      "step": 145,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.3671801320339516,
      "grad_norm": 3.0926783084869385,
      "learning_rate": 0.00019716028209058387,
      "logits/chosen": -0.5261669158935547,
      "logits/rejected": -0.6899480223655701,
      "logps/chosen": -6.620856285095215,
      "logps/rejected": -44.43523406982422,
      "loss": 0.9013221263885498,
      "memory(GiB)": 42.61,
      "nll_loss": 0.658555269241333,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.10268372297286987,
      "rewards/margins": 2.719848155975342,
      "rewards/rejected": -2.8225317001342773,
      "step": 146,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.36969506444514305,
      "grad_norm": 1.952876329421997,
      "learning_rate": 0.0001970941817426052,
      "logits/chosen": -0.3797496557235718,
      "logits/rejected": -0.6095885038375854,
      "logps/chosen": -3.3423259258270264,
      "logps/rejected": -51.2366943359375,
      "loss": 0.3124345541000366,
      "memory(GiB)": 42.61,
      "nll_loss": 0.19474804401397705,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15071216225624084,
      "rewards/margins": 3.3171157836914062,
      "rewards/rejected": -3.166403293609619,
      "step": 147,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.3722099968563345,
      "grad_norm": 5.522589683532715,
      "learning_rate": 0.00019702733224690605,
      "logits/chosen": -0.4721255898475647,
      "logits/rejected": -0.6467065215110779,
      "logps/chosen": -8.671660423278809,
      "logps/rejected": -41.544979095458984,
      "loss": 0.7880657911300659,
      "memory(GiB)": 42.61,
      "nll_loss": 0.553983211517334,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06357112526893616,
      "rewards/margins": 2.7004964351654053,
      "rewards/rejected": -2.7640676498413086,
      "step": 148,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.37472492926752593,
      "grad_norm": 1.7939379215240479,
      "learning_rate": 0.00019695973411927578,
      "logits/chosen": -0.518312394618988,
      "logits/rejected": -0.689118504524231,
      "logps/chosen": -2.949949026107788,
      "logps/rejected": -47.91648864746094,
      "loss": 0.3869367241859436,
      "memory(GiB)": 42.61,
      "nll_loss": 0.22588227689266205,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.06105111539363861,
      "rewards/margins": 3.0687060356140137,
      "rewards/rejected": -3.0076546669006348,
      "step": 149,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.37723986167871737,
      "grad_norm": 1.6997706890106201,
      "learning_rate": 0.00019689138788127995,
      "logits/chosen": -0.49549129605293274,
      "logits/rejected": -0.6151180267333984,
      "logps/chosen": -6.0540666580200195,
      "logps/rejected": -35.12141036987305,
      "loss": 0.6853822469711304,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4531955420970917,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.015824541449546814,
      "rewards/margins": 2.214078903198242,
      "rewards/rejected": -2.198254346847534,
      "step": 150,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.3797547940899088,
      "grad_norm": 1.94599449634552,
      "learning_rate": 0.00019682229406025635,
      "logits/chosen": -0.44517412781715393,
      "logits/rejected": -0.583071231842041,
      "logps/chosen": -8.055398941040039,
      "logps/rejected": -40.7237548828125,
      "loss": 0.6454893350601196,
      "memory(GiB)": 42.61,
      "nll_loss": 0.38740435242652893,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09544569253921509,
      "rewards/margins": 2.4482340812683105,
      "rewards/rejected": -2.3527884483337402,
      "step": 151,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.3822697265011003,
      "grad_norm": 1.4162402153015137,
      "learning_rate": 0.00019675245318931083,
      "logits/chosen": -0.5134773254394531,
      "logits/rejected": -0.6419417858123779,
      "logps/chosen": -9.19031047821045,
      "logps/rejected": -41.0048828125,
      "loss": 0.6307085752487183,
      "memory(GiB)": 42.61,
      "nll_loss": 0.42102259397506714,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.08155234903097153,
      "rewards/margins": 2.483502149581909,
      "rewards/rejected": -2.565054416656494,
      "step": 152,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.38478465891229174,
      "grad_norm": 1.932735562324524,
      "learning_rate": 0.0001966818658073133,
      "logits/chosen": -0.5457805395126343,
      "logits/rejected": -0.680858850479126,
      "logps/chosen": -6.187590599060059,
      "logps/rejected": -37.30434036254883,
      "loss": 0.6069836020469666,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3702396750450134,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.003183305263519287,
      "rewards/margins": 2.3362035751342773,
      "rewards/rejected": -2.3330204486846924,
      "step": 153,
      "train_speed(iter/s)": 0.011222
    },
    {
      "epoch": 0.3872995913234832,
      "grad_norm": 2.0515635013580322,
      "learning_rate": 0.0001966105324588934,
      "logits/chosen": -0.5253856182098389,
      "logits/rejected": -0.6266660094261169,
      "logps/chosen": -11.048845291137695,
      "logps/rejected": -40.014915466308594,
      "loss": 0.8203007578849792,
      "memory(GiB)": 42.61,
      "nll_loss": 0.56341153383255,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.09865356236696243,
      "rewards/margins": 2.6950490474700928,
      "rewards/rejected": -2.596395492553711,
      "step": 154,
      "train_speed(iter/s)": 0.011222
    },
    {
      "epoch": 0.3898145237346746,
      "grad_norm": 2.258105516433716,
      "learning_rate": 0.00019653845369443657,
      "logits/chosen": -0.49116846919059753,
      "logits/rejected": -0.6502944231033325,
      "logps/chosen": -6.3410539627075195,
      "logps/rejected": -45.01954650878906,
      "loss": 0.6601016521453857,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4770495295524597,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08940570056438446,
      "rewards/margins": 2.79660701751709,
      "rewards/rejected": -2.7072014808654785,
      "step": 155,
      "train_speed(iter/s)": 0.011222
    },
    {
      "epoch": 0.39232945614586606,
      "grad_norm": 1.9334651231765747,
      "learning_rate": 0.00019646563007007952,
      "logits/chosen": -0.5132689476013184,
      "logits/rejected": -0.6826451420783997,
      "logps/chosen": -4.647260665893555,
      "logps/rejected": -35.987144470214844,
      "loss": 0.5781624913215637,
      "memory(GiB)": 42.61,
      "nll_loss": 0.333342581987381,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.0551791712641716,
      "rewards/margins": 1.9761953353881836,
      "rewards/rejected": -1.9210160970687866,
      "step": 156,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.3948443885570575,
      "grad_norm": 2.0030107498168945,
      "learning_rate": 0.00019639206214770608,
      "logits/chosen": -0.5250845551490784,
      "logits/rejected": -0.6993058919906616,
      "logps/chosen": -3.4496142864227295,
      "logps/rejected": -45.091514587402344,
      "loss": 0.4796870946884155,
      "memory(GiB)": 42.61,
      "nll_loss": 0.27256864309310913,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.039092980325222015,
      "rewards/margins": 2.82478666305542,
      "rewards/rejected": -2.785693645477295,
      "step": 157,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.397359320968249,
      "grad_norm": 1.6849418878555298,
      "learning_rate": 0.00019631775049494285,
      "logits/chosen": -0.43024155497550964,
      "logits/rejected": -0.6700482964515686,
      "logps/chosen": -3.823197364807129,
      "logps/rejected": -56.609588623046875,
      "loss": 0.42021045088768005,
      "memory(GiB)": 42.61,
      "nll_loss": 0.24996069073677063,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.004662778228521347,
      "rewards/margins": 3.1713931560516357,
      "rewards/rejected": -3.1667304039001465,
      "step": 158,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.39987425337944044,
      "grad_norm": 3.1281967163085938,
      "learning_rate": 0.00019624269568515486,
      "logits/chosen": -0.43441203236579895,
      "logits/rejected": -0.6460458040237427,
      "logps/chosen": -6.727141857147217,
      "logps/rejected": -45.23570251464844,
      "loss": 0.7260392904281616,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5018711090087891,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.011816263198852539,
      "rewards/margins": 2.4935953617095947,
      "rewards/rejected": -2.4817793369293213,
      "step": 159,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.4023891857906319,
      "grad_norm": 3.3100078105926514,
      "learning_rate": 0.00019616689829744105,
      "logits/chosen": -0.6007479429244995,
      "logits/rejected": -0.7636129856109619,
      "logps/chosen": -6.275008678436279,
      "logps/rejected": -35.27611541748047,
      "loss": 0.6686663627624512,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4275904893875122,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1287657767534256,
      "rewards/margins": 2.2340850830078125,
      "rewards/rejected": -2.1053192615509033,
      "step": 160,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.4049041182018233,
      "grad_norm": 3.4858033657073975,
      "learning_rate": 0.0001960903589166299,
      "logits/chosen": -0.6405993700027466,
      "logits/rejected": -0.697019100189209,
      "logps/chosen": -9.289031982421875,
      "logps/rejected": -38.57951736450195,
      "loss": 0.7603179216384888,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4232831597328186,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.159571573138237,
      "rewards/margins": 2.720222234725952,
      "rewards/rejected": -2.56065034866333,
      "step": 161,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.40741905061301475,
      "grad_norm": 1.844115972518921,
      "learning_rate": 0.00019601307813327482,
      "logits/chosen": -0.5259416103363037,
      "logits/rejected": -0.6975188255310059,
      "logps/chosen": -2.951578140258789,
      "logps/rejected": -41.11060333251953,
      "loss": 0.4498739242553711,
      "memory(GiB)": 42.61,
      "nll_loss": 0.23409336805343628,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14196570217609406,
      "rewards/margins": 2.5788159370422363,
      "rewards/rejected": -2.436850070953369,
      "step": 162,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.40993398302420625,
      "grad_norm": 5.637467861175537,
      "learning_rate": 0.00019593505654364965,
      "logits/chosen": -0.45152366161346436,
      "logits/rejected": -0.6798401474952698,
      "logps/chosen": -4.170862674713135,
      "logps/rejected": -53.169776916503906,
      "loss": 0.6846017241477966,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5021944642066956,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.010230405256152153,
      "rewards/margins": 3.3515424728393555,
      "rewards/rejected": -3.3413121700286865,
      "step": 163,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.4124489154353977,
      "grad_norm": 1.4745384454727173,
      "learning_rate": 0.00019585629474974415,
      "logits/chosen": -0.4008867144584656,
      "logits/rejected": -0.7353089451789856,
      "logps/chosen": -0.4037323594093323,
      "logps/rejected": -61.83756637573242,
      "loss": 0.18311206996440887,
      "memory(GiB)": 42.61,
      "nll_loss": 0.05691269040107727,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09548275917768478,
      "rewards/margins": 4.11773157119751,
      "rewards/rejected": -4.0222487449646,
      "step": 164,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.41496384784658913,
      "grad_norm": 1.9870681762695312,
      "learning_rate": 0.0001957767933592591,
      "logits/chosen": -0.5013211965560913,
      "logits/rejected": -0.6647384762763977,
      "logps/chosen": -6.108734130859375,
      "logps/rejected": -45.8098030090332,
      "loss": 0.6409457921981812,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3451662063598633,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.025398313999176025,
      "rewards/margins": 3.0071825981140137,
      "rewards/rejected": -3.032581090927124,
      "step": 165,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.41747878025778057,
      "grad_norm": 3.7264280319213867,
      "learning_rate": 0.00019569655298560195,
      "logits/chosen": -0.5106596350669861,
      "logits/rejected": -0.7329829931259155,
      "logps/chosen": -4.24332857131958,
      "logps/rejected": -48.86112594604492,
      "loss": 0.5813378691673279,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3752741515636444,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06557303667068481,
      "rewards/margins": 3.4898738861083984,
      "rewards/rejected": -3.4243011474609375,
      "step": 166,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.419993712668972,
      "grad_norm": 11.64145278930664,
      "learning_rate": 0.0001956155742478817,
      "logits/chosen": -0.4935857653617859,
      "logits/rejected": -0.7044666409492493,
      "logps/chosen": -2.654214382171631,
      "logps/rejected": -57.629425048828125,
      "loss": 0.3905934691429138,
      "memory(GiB)": 42.61,
      "nll_loss": 0.22700335085391998,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1508655548095703,
      "rewards/margins": 4.179229736328125,
      "rewards/rejected": -4.028364181518555,
      "step": 167,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.42250864508016345,
      "grad_norm": 1.6378587484359741,
      "learning_rate": 0.00019553385777090458,
      "logits/chosen": -0.4575374722480774,
      "logits/rejected": -0.6927694082260132,
      "logps/chosen": -9.244451522827148,
      "logps/rejected": -63.81715774536133,
      "loss": 0.6180571913719177,
      "memory(GiB)": 44.38,
      "nll_loss": 0.47105246782302856,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12050072848796844,
      "rewards/margins": 4.137886047363281,
      "rewards/rejected": -4.017385005950928,
      "step": 168,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.42502357749135494,
      "grad_norm": 6.146910667419434,
      "learning_rate": 0.00019545140418516873,
      "logits/chosen": -0.6766417622566223,
      "logits/rejected": -0.7711372971534729,
      "logps/chosen": -6.37576961517334,
      "logps/rejected": -45.8558349609375,
      "loss": 0.6669228672981262,
      "memory(GiB)": 44.38,
      "nll_loss": 0.5145984888076782,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18157291412353516,
      "rewards/margins": 3.766644239425659,
      "rewards/rejected": -3.585071563720703,
      "step": 169,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.4275385099025464,
      "grad_norm": 1.6984812021255493,
      "learning_rate": 0.0001953682141268598,
      "logits/chosen": -0.5011512637138367,
      "logits/rejected": -0.688590943813324,
      "logps/chosen": -3.602705478668213,
      "logps/rejected": -55.99202346801758,
      "loss": 0.3990403413772583,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2418036162853241,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.027019452303647995,
      "rewards/margins": 3.8597612380981445,
      "rewards/rejected": -3.8327417373657227,
      "step": 170,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.4300534423137378,
      "grad_norm": 5.023574352264404,
      "learning_rate": 0.00019528428823784567,
      "logits/chosen": -0.4159168004989624,
      "logits/rejected": -0.7052143216133118,
      "logps/chosen": -2.2552971839904785,
      "logps/rejected": -71.4410400390625,
      "loss": 0.29148706793785095,
      "memory(GiB)": 44.38,
      "nll_loss": 0.19378283619880676,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13933303952217102,
      "rewards/margins": 5.2820634841918945,
      "rewards/rejected": -5.142730236053467,
      "step": 171,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.43256837472492926,
      "grad_norm": 1.4866183996200562,
      "learning_rate": 0.0001951996271656717,
      "logits/chosen": -0.5606605410575867,
      "logits/rejected": -0.7212972044944763,
      "logps/chosen": -3.804295539855957,
      "logps/rejected": -51.83647537231445,
      "loss": 0.3623957335948944,
      "memory(GiB)": 44.38,
      "nll_loss": 0.21611489355564117,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12722836434841156,
      "rewards/margins": 3.714401960372925,
      "rewards/rejected": -3.5871734619140625,
      "step": 172,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.4350833071361207,
      "grad_norm": 3.7671830654144287,
      "learning_rate": 0.00019511423156355577,
      "logits/chosen": -0.4943828582763672,
      "logits/rejected": -0.68037348985672,
      "logps/chosen": -5.188858509063721,
      "logps/rejected": -62.85913848876953,
      "loss": 0.41962915658950806,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2764420509338379,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1342807114124298,
      "rewards/margins": 4.525426387786865,
      "rewards/rejected": -4.391145706176758,
      "step": 173,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.43759823954731214,
      "grad_norm": 9.907013893127441,
      "learning_rate": 0.00019502810209038303,
      "logits/chosen": -0.5515968799591064,
      "logits/rejected": -0.6605091691017151,
      "logps/chosen": -9.89529800415039,
      "logps/rejected": -54.34276580810547,
      "loss": 0.9658512473106384,
      "memory(GiB)": 44.38,
      "nll_loss": 0.7056573033332825,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.028916141018271446,
      "rewards/margins": 3.5635647773742676,
      "rewards/rejected": -3.5924811363220215,
      "step": 174,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.44011317195850364,
      "grad_norm": 2.1124749183654785,
      "learning_rate": 0.00019494123941070108,
      "logits/chosen": -0.689885675907135,
      "logits/rejected": -0.7663485407829285,
      "logps/chosen": -5.329808712005615,
      "logps/rejected": -39.43062210083008,
      "loss": 0.6925463676452637,
      "memory(GiB)": 44.38,
      "nll_loss": 0.473371297121048,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10141712427139282,
      "rewards/margins": 3.0223309993743896,
      "rewards/rejected": -2.9209136962890625,
      "step": 175,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.4426281043696951,
      "grad_norm": 36.679805755615234,
      "learning_rate": 0.00019485364419471454,
      "logits/chosen": -0.6628389954566956,
      "logits/rejected": -0.7602969408035278,
      "logps/chosen": -7.741974353790283,
      "logps/rejected": -50.05625915527344,
      "loss": 1.0915241241455078,
      "memory(GiB)": 44.38,
      "nll_loss": 0.7456662654876709,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.35289716720581055,
      "rewards/margins": 3.5574560165405273,
      "rewards/rejected": -3.910353422164917,
      "step": 176,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.4451430367808865,
      "grad_norm": 2.4835803508758545,
      "learning_rate": 0.00019476531711828027,
      "logits/chosen": -0.5590236186981201,
      "logits/rejected": -0.6718421578407288,
      "logps/chosen": -5.849733352661133,
      "logps/rejected": -51.53736114501953,
      "loss": 0.40270960330963135,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2558823227882385,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11730007827281952,
      "rewards/margins": 3.6312365531921387,
      "rewards/rejected": -3.5139362812042236,
      "step": 177,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.44765796919207795,
      "grad_norm": 2.382329225540161,
      "learning_rate": 0.00019467625886290167,
      "logits/chosen": -0.5211893916130066,
      "logits/rejected": -0.6878003478050232,
      "logps/chosen": -11.300158500671387,
      "logps/rejected": -52.15276336669922,
      "loss": 0.7076267004013062,
      "memory(GiB)": 44.38,
      "nll_loss": 0.5078485012054443,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07471176981925964,
      "rewards/margins": 3.560542583465576,
      "rewards/rejected": -3.485830783843994,
      "step": 178,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.4501729016032694,
      "grad_norm": 4.822922706604004,
      "learning_rate": 0.0001945864701157239,
      "logits/chosen": -0.5776659250259399,
      "logits/rejected": -0.6799846291542053,
      "logps/chosen": -4.035768032073975,
      "logps/rejected": -54.64043426513672,
      "loss": 0.48212817311286926,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2608688175678253,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17263993620872498,
      "rewards/margins": 4.305383682250977,
      "rewards/rejected": -4.132743835449219,
      "step": 179,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.4526878340144609,
      "grad_norm": 1.466308832168579,
      "learning_rate": 0.00019449595156952827,
      "logits/chosen": -0.6198251247406006,
      "logits/rejected": -0.7188805341720581,
      "logps/chosen": -7.5114359855651855,
      "logps/rejected": -50.14148712158203,
      "loss": 0.5570878982543945,
      "memory(GiB)": 44.38,
      "nll_loss": 0.3523883819580078,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.18179449439048767,
      "rewards/margins": 3.7667787075042725,
      "rewards/rejected": -3.584984302520752,
      "step": 180,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.4552027664256523,
      "grad_norm": 1.5548723936080933,
      "learning_rate": 0.00019440470392272696,
      "logits/chosen": -0.5984869003295898,
      "logits/rejected": -0.7039155960083008,
      "logps/chosen": -4.1996378898620605,
      "logps/rejected": -45.629638671875,
      "loss": 0.47419819235801697,
      "memory(GiB)": 44.38,
      "nll_loss": 0.31078362464904785,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1542869210243225,
      "rewards/margins": 3.5583298206329346,
      "rewards/rejected": -3.404043197631836,
      "step": 181,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.45771769883684377,
      "grad_norm": 2.250375747680664,
      "learning_rate": 0.00019431272787935773,
      "logits/chosen": -0.5019629597663879,
      "logits/rejected": -0.7356317639350891,
      "logps/chosen": -10.908112525939941,
      "logps/rejected": -52.47066116333008,
      "loss": 1.0049289464950562,
      "memory(GiB)": 44.38,
      "nll_loss": 0.7764756083488464,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.024810096248984337,
      "rewards/margins": 3.2386226654052734,
      "rewards/rejected": -3.213812828063965,
      "step": 182,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.4602326312480352,
      "grad_norm": 22.676647186279297,
      "learning_rate": 0.00019422002414907837,
      "logits/chosen": -0.5647466778755188,
      "logits/rejected": -0.7478159070014954,
      "logps/chosen": -5.912107944488525,
      "logps/rejected": -47.091400146484375,
      "loss": 0.699102520942688,
      "memory(GiB)": 44.38,
      "nll_loss": 0.4706324338912964,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08286911249160767,
      "rewards/margins": 3.223126173019409,
      "rewards/rejected": -3.140256881713867,
      "step": 183,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.46274756365922665,
      "grad_norm": 3.388159990310669,
      "learning_rate": 0.00019412659344716124,
      "logits/chosen": -0.5706430673599243,
      "logits/rejected": -0.6877136826515198,
      "logps/chosen": -11.683382987976074,
      "logps/rejected": -46.81083297729492,
      "loss": 1.3097541332244873,
      "memory(GiB)": 44.38,
      "nll_loss": 0.8712559342384338,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.2759455144405365,
      "rewards/margins": 3.09993839263916,
      "rewards/rejected": -3.3758838176727295,
      "step": 184,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.4652624960704181,
      "grad_norm": 2.6059823036193848,
      "learning_rate": 0.00019403243649448787,
      "logits/chosen": -0.47367703914642334,
      "logits/rejected": -0.7172597050666809,
      "logps/chosen": -3.135293483734131,
      "logps/rejected": -64.97146606445312,
      "loss": 0.4278358221054077,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2768169641494751,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1706126630306244,
      "rewards/margins": 4.8074727058410645,
      "rewards/rejected": -4.636859893798828,
      "step": 185,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.4677774284816096,
      "grad_norm": 2.2509593963623047,
      "learning_rate": 0.00019393755401754322,
      "logits/chosen": -0.48067784309387207,
      "logits/rejected": -0.6601268649101257,
      "logps/chosen": -3.7737672328948975,
      "logps/rejected": -61.828006744384766,
      "loss": 0.44278520345687866,
      "memory(GiB)": 44.38,
      "nll_loss": 0.3135760426521301,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.05582159385085106,
      "rewards/margins": 4.652613162994385,
      "rewards/rejected": -4.5967912673950195,
      "step": 186,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.470292360892801,
      "grad_norm": 2.220067262649536,
      "learning_rate": 0.00019384194674841026,
      "logits/chosen": -0.5368369221687317,
      "logits/rejected": -0.7164134383201599,
      "logps/chosen": -4.382404804229736,
      "logps/rejected": -63.785423278808594,
      "loss": 0.5859192609786987,
      "memory(GiB)": 44.38,
      "nll_loss": 0.4215310215950012,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.042068447917699814,
      "rewards/margins": 5.157468795776367,
      "rewards/rejected": -5.115400314331055,
      "step": 187,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.47280729330399246,
      "grad_norm": 1.3244057893753052,
      "learning_rate": 0.0001937456154247641,
      "logits/chosen": -0.5555019974708557,
      "logits/rejected": -0.682890772819519,
      "logps/chosen": -4.023111343383789,
      "logps/rejected": -66.17463684082031,
      "loss": 0.3469849228858948,
      "memory(GiB)": 44.38,
      "nll_loss": 0.22850053012371063,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.08244925737380981,
      "rewards/margins": 5.2128143310546875,
      "rewards/rejected": -5.130365371704102,
      "step": 188,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.4753222257151839,
      "grad_norm": 5.5003886222839355,
      "learning_rate": 0.00019364856078986652,
      "logits/chosen": -0.504755437374115,
      "logits/rejected": -0.6091613173484802,
      "logps/chosen": -13.118067741394043,
      "logps/rejected": -52.780235290527344,
      "loss": 0.970127284526825,
      "memory(GiB)": 44.38,
      "nll_loss": 0.7232432961463928,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04733971506357193,
      "rewards/margins": 3.863572120666504,
      "rewards/rejected": -3.910911798477173,
      "step": 189,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.47783715812637534,
      "grad_norm": 2.35336971282959,
      "learning_rate": 0.0001935507835925601,
      "logits/chosen": -0.4966239631175995,
      "logits/rejected": -0.6422093510627747,
      "logps/chosen": -5.089540004730225,
      "logps/rejected": -60.03197479248047,
      "loss": 0.48357081413269043,
      "memory(GiB)": 44.38,
      "nll_loss": 0.29401686787605286,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06470837444067001,
      "rewards/margins": 4.633599758148193,
      "rewards/rejected": -4.568891525268555,
      "step": 190,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.4803520905375668,
      "grad_norm": 2.2711422443389893,
      "learning_rate": 0.00019345228458726252,
      "logits/chosen": -0.41524171829223633,
      "logits/rejected": -0.5779014229774475,
      "logps/chosen": -6.11699914932251,
      "logps/rejected": -65.24181365966797,
      "loss": 0.5390986800193787,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2827398180961609,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04118676483631134,
      "rewards/margins": 4.437079906463623,
      "rewards/rejected": -4.395893096923828,
      "step": 191,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.4828670229487583,
      "grad_norm": 3.4174418449401855,
      "learning_rate": 0.00019335306453396064,
      "logits/chosen": -0.3581904470920563,
      "logits/rejected": -0.5604187250137329,
      "logps/chosen": -4.7871503829956055,
      "logps/rejected": -56.127437591552734,
      "loss": 0.6041537523269653,
      "memory(GiB)": 44.38,
      "nll_loss": 0.35478317737579346,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0018234718590974808,
      "rewards/margins": 3.8149027824401855,
      "rewards/rejected": -3.8130791187286377,
      "step": 192,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.4853819553599497,
      "grad_norm": 1.6018799543380737,
      "learning_rate": 0.00019325312419820473,
      "logits/chosen": -0.40660223364830017,
      "logits/rejected": -0.5828196406364441,
      "logps/chosen": -4.125640869140625,
      "logps/rejected": -52.407806396484375,
      "loss": 0.4834650158882141,
      "memory(GiB)": 44.38,
      "nll_loss": 0.2773113250732422,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08193957805633545,
      "rewards/margins": 3.58589506149292,
      "rewards/rejected": -3.503955602645874,
      "step": 193,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.48789688777114115,
      "grad_norm": 2.956265926361084,
      "learning_rate": 0.00019315246435110256,
      "logits/chosen": -0.41891956329345703,
      "logits/rejected": -0.5709394812583923,
      "logps/chosen": -6.136990547180176,
      "logps/rejected": -54.53316116333008,
      "loss": 0.5783406496047974,
      "memory(GiB)": 44.38,
      "nll_loss": 0.4396575689315796,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22025173902511597,
      "rewards/margins": 3.940415143966675,
      "rewards/rejected": -3.7201638221740723,
      "step": 194,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.4904118201823326,
      "grad_norm": 2.4830644130706787,
      "learning_rate": 0.00019305108576931336,
      "logits/chosen": -0.4251477122306824,
      "logits/rejected": -0.5646131038665771,
      "logps/chosen": -6.740220069885254,
      "logps/rejected": -59.58277893066406,
      "loss": 0.6179866194725037,
      "memory(GiB)": 44.38,
      "nll_loss": 0.39784371852874756,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.03337670490145683,
      "rewards/margins": 4.295322418212891,
      "rewards/rejected": -4.261946201324463,
      "step": 195,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.49292675259352403,
      "grad_norm": 2.0805859565734863,
      "learning_rate": 0.0001929489892350419,
      "logits/chosen": -0.4012482464313507,
      "logits/rejected": -0.5744894742965698,
      "logps/chosen": -3.9474775791168213,
      "logps/rejected": -64.57502746582031,
      "loss": 0.4640367329120636,
      "memory(GiB)": 44.38,
      "nll_loss": 0.32923948764801025,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11397141218185425,
      "rewards/margins": 4.956214904785156,
      "rewards/rejected": -4.842243194580078,
      "step": 196,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.49544168500471547,
      "grad_norm": 1.0079342126846313,
      "learning_rate": 0.00019284617553603242,
      "logits/chosen": -0.40687650442123413,
      "logits/rejected": -0.5095179677009583,
      "logps/chosen": -5.0278239250183105,
      "logps/rejected": -57.115501403808594,
      "loss": 0.5296419262886047,
      "memory(GiB)": 44.38,
      "nll_loss": 0.3394710123538971,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.223480224609375,
      "rewards/margins": 4.145529270172119,
      "rewards/rejected": -3.9220495223999023,
      "step": 197,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.49795661741590697,
      "grad_norm": 4.113004207611084,
      "learning_rate": 0.0001927426454655627,
      "logits/chosen": -0.35911667346954346,
      "logits/rejected": -0.5175841450691223,
      "logps/chosen": -7.666455268859863,
      "logps/rejected": -67.4910659790039,
      "loss": 0.5694496035575867,
      "memory(GiB)": 44.38,
      "nll_loss": 0.4178715944290161,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.04037248343229294,
      "rewards/margins": 4.807923793792725,
      "rewards/rejected": -4.848296165466309,
      "step": 198,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.5004715498270984,
      "grad_norm": 0.9368858933448792,
      "learning_rate": 0.0001926383998224377,
      "logits/chosen": -0.3456733524799347,
      "logits/rejected": -0.5820738077163696,
      "logps/chosen": -1.7521214485168457,
      "logps/rejected": -69.5865249633789,
      "loss": 0.225211039185524,
      "memory(GiB)": 46.14,
      "nll_loss": 0.13497470319271088,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.19091293215751648,
      "rewards/margins": 5.2458882331848145,
      "rewards/rejected": -5.0549750328063965,
      "step": 199,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.5029864822382899,
      "grad_norm": 2.017502784729004,
      "learning_rate": 0.0001925334394109835,
      "logits/chosen": -0.34290656447410583,
      "logits/rejected": -0.5616070032119751,
      "logps/chosen": -5.75649881362915,
      "logps/rejected": -66.36112213134766,
      "loss": 0.5342864990234375,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3393324017524719,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.017528314143419266,
      "rewards/margins": 4.61277437210083,
      "rewards/rejected": -4.630302906036377,
      "step": 200,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.5055014146494813,
      "grad_norm": 11.226442337036133,
      "learning_rate": 0.00019242776504104118,
      "logits/chosen": -0.35404524207115173,
      "logits/rejected": -0.5164979696273804,
      "logps/chosen": -6.239109992980957,
      "logps/rejected": -57.19633102416992,
      "loss": 0.751146674156189,
      "memory(GiB)": 46.14,
      "nll_loss": 0.5300025939941406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03473217412829399,
      "rewards/margins": 4.055646896362305,
      "rewards/rejected": -4.020914554595947,
      "step": 201,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.5080163470606728,
      "grad_norm": 2.073298454284668,
      "learning_rate": 0.00019232137752796044,
      "logits/chosen": -0.4284801483154297,
      "logits/rejected": -0.5613614916801453,
      "logps/chosen": -4.676477909088135,
      "logps/rejected": -53.5533447265625,
      "loss": 0.6112902164459229,
      "memory(GiB)": 46.14,
      "nll_loss": 0.47125840187072754,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11576966941356659,
      "rewards/margins": 3.918217420578003,
      "rewards/rejected": -3.80244779586792,
      "step": 202,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.5105312794718642,
      "grad_norm": 1.5557101964950562,
      "learning_rate": 0.00019221427769259333,
      "logits/chosen": -0.34159645438194275,
      "logits/rejected": -0.539400577545166,
      "logps/chosen": -3.861686944961548,
      "logps/rejected": -61.914703369140625,
      "loss": 0.40771356225013733,
      "memory(GiB)": 46.14,
      "nll_loss": 0.29954829812049866,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2123047411441803,
      "rewards/margins": 4.397878646850586,
      "rewards/rejected": -4.185573577880859,
      "step": 203,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.5130462118830557,
      "grad_norm": 9.60893726348877,
      "learning_rate": 0.00019210646636128807,
      "logits/chosen": -0.37730276584625244,
      "logits/rejected": -0.4876512885093689,
      "logps/chosen": -4.578537464141846,
      "logps/rejected": -47.36624526977539,
      "loss": 0.45369648933410645,
      "memory(GiB)": 46.14,
      "nll_loss": 0.274221271276474,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08797872066497803,
      "rewards/margins": 3.566829204559326,
      "rewards/rejected": -3.4788498878479004,
      "step": 204,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.515561144294247,
      "grad_norm": 1.5941282510757446,
      "learning_rate": 0.00019199794436588243,
      "logits/chosen": -0.27956530451774597,
      "logits/rejected": -0.4699462652206421,
      "logps/chosen": -7.073207378387451,
      "logps/rejected": -68.29771423339844,
      "loss": 0.3854275047779083,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2786342203617096,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12901465594768524,
      "rewards/margins": 4.810124397277832,
      "rewards/rejected": -4.68110990524292,
      "step": 205,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.5180760767054385,
      "grad_norm": 5.089592456817627,
      "learning_rate": 0.00019188871254369752,
      "logits/chosen": -0.31783169507980347,
      "logits/rejected": -0.427703857421875,
      "logps/chosen": -5.960029602050781,
      "logps/rejected": -43.00883102416992,
      "loss": 0.8745622634887695,
      "memory(GiB)": 46.14,
      "nll_loss": 0.5852820873260498,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.11267540603876114,
      "rewards/margins": 2.461634635925293,
      "rewards/rejected": -2.574310064315796,
      "step": 206,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.52059100911663,
      "grad_norm": 2.929906129837036,
      "learning_rate": 0.00019177877173753128,
      "logits/chosen": -0.35540899634361267,
      "logits/rejected": -0.4668577313423157,
      "logps/chosen": -5.586899757385254,
      "logps/rejected": -46.6971435546875,
      "loss": 0.5985509753227234,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3464246988296509,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.003932179883122444,
      "rewards/margins": 3.3567466735839844,
      "rewards/rejected": -3.3606786727905273,
      "step": 207,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.5231059415278214,
      "grad_norm": 1.8040342330932617,
      "learning_rate": 0.00019166812279565178,
      "logits/chosen": -0.3284756541252136,
      "logits/rejected": -0.43804702162742615,
      "logps/chosen": -5.879787445068359,
      "logps/rejected": -52.28423309326172,
      "loss": 0.6572303175926208,
      "memory(GiB)": 46.14,
      "nll_loss": 0.46364596486091614,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10764454305171967,
      "rewards/margins": 3.84014892578125,
      "rewards/rejected": -3.732503890991211,
      "step": 208,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.5256208739390129,
      "grad_norm": 2.197678565979004,
      "learning_rate": 0.000191556766571791,
      "logits/chosen": -0.2658138871192932,
      "logits/rejected": -0.4723971486091614,
      "logps/chosen": -6.082084655761719,
      "logps/rejected": -70.72124481201172,
      "loss": 0.563379168510437,
      "memory(GiB)": 46.14,
      "nll_loss": 0.4301711916923523,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.03504669666290283,
      "rewards/margins": 4.98071813583374,
      "rewards/rejected": -5.0157647132873535,
      "step": 209,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.5281358063502043,
      "grad_norm": 1.8066035509109497,
      "learning_rate": 0.000191444703925138,
      "logits/chosen": -0.266013503074646,
      "logits/rejected": -0.4869195520877838,
      "logps/chosen": -3.5636188983917236,
      "logps/rejected": -65.75146484375,
      "loss": 0.41913098096847534,
      "memory(GiB)": 46.14,
      "nll_loss": 0.29366493225097656,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.20083719491958618,
      "rewards/margins": 4.579287052154541,
      "rewards/rejected": -4.3784499168396,
      "step": 210,
      "train_speed(iter/s)": 0.011225
    },
    {
      "epoch": 0.5306507387613958,
      "grad_norm": 2.058502197265625,
      "learning_rate": 0.00019133193572033242,
      "logits/chosen": -0.2633608877658844,
      "logits/rejected": -0.4534919857978821,
      "logps/chosen": -5.140444278717041,
      "logps/rejected": -65.33726501464844,
      "loss": 0.4311324954032898,
      "memory(GiB)": 46.14,
      "nll_loss": 0.25075870752334595,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.013513857498764992,
      "rewards/margins": 4.880782127380371,
      "rewards/rejected": -4.894296169281006,
      "step": 211,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5331656711725873,
      "grad_norm": 4.458026885986328,
      "learning_rate": 0.00019121846282745778,
      "logits/chosen": -0.31052568554878235,
      "logits/rejected": -0.40828073024749756,
      "logps/chosen": -6.845982074737549,
      "logps/rejected": -55.935951232910156,
      "loss": 0.6287424564361572,
      "memory(GiB)": 46.14,
      "nll_loss": 0.47837764024734497,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.0341886430978775,
      "rewards/margins": 4.157693862915039,
      "rewards/rejected": -4.191883087158203,
      "step": 212,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5356806035837787,
      "grad_norm": 1.700737476348877,
      "learning_rate": 0.00019110428612203464,
      "logits/chosen": -0.3173929452896118,
      "logits/rejected": -0.49177926778793335,
      "logps/chosen": -6.933238506317139,
      "logps/rejected": -63.277774810791016,
      "loss": 0.48178917169570923,
      "memory(GiB)": 46.14,
      "nll_loss": 0.32791826128959656,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.009786809794604778,
      "rewards/margins": 4.001126289367676,
      "rewards/rejected": -3.991339683532715,
      "step": 213,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5381955359949702,
      "grad_norm": 2.141183614730835,
      "learning_rate": 0.00019098940648501405,
      "logits/chosen": -0.34126630425453186,
      "logits/rejected": -0.5091477036476135,
      "logps/chosen": -2.5000686645507812,
      "logps/rejected": -56.68462371826172,
      "loss": 0.26290345191955566,
      "memory(GiB)": 46.14,
      "nll_loss": 0.13039958477020264,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0922369658946991,
      "rewards/margins": 4.214053153991699,
      "rewards/rejected": -4.1218156814575195,
      "step": 214,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5407104684061615,
      "grad_norm": 1.0281932353973389,
      "learning_rate": 0.0001908738248027706,
      "logits/chosen": -0.31923872232437134,
      "logits/rejected": -0.5267022252082825,
      "logps/chosen": -3.376938819885254,
      "logps/rejected": -68.53611755371094,
      "loss": 0.3598852753639221,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2368268519639969,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13397309184074402,
      "rewards/margins": 5.190869331359863,
      "rewards/rejected": -5.056896209716797,
      "step": 215,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.543225400817353,
      "grad_norm": 1.979490876197815,
      "learning_rate": 0.00019075754196709572,
      "logits/chosen": -0.3671177327632904,
      "logits/rejected": -0.5938350558280945,
      "logps/chosen": -2.393630266189575,
      "logps/rejected": -76.2723388671875,
      "loss": 0.3168295919895172,
      "memory(GiB)": 46.14,
      "nll_loss": 0.20362113416194916,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1339578777551651,
      "rewards/margins": 5.975349426269531,
      "rewards/rejected": -5.8413920402526855,
      "step": 216,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5457403332285444,
      "grad_norm": 7.617611408233643,
      "learning_rate": 0.00019064055887519062,
      "logits/chosen": -0.4243953227996826,
      "logits/rejected": -0.5465304851531982,
      "logps/chosen": -8.081028938293457,
      "logps/rejected": -70.03741455078125,
      "loss": 0.7748145461082458,
      "memory(GiB)": 46.14,
      "nll_loss": 0.5802915096282959,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.02728523313999176,
      "rewards/margins": 5.3238701820373535,
      "rewards/rejected": -5.3511552810668945,
      "step": 217,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.5482552656397359,
      "grad_norm": 2.162104845046997,
      "learning_rate": 0.00019052287642965952,
      "logits/chosen": -0.40015673637390137,
      "logits/rejected": -0.5828264355659485,
      "logps/chosen": -5.614627838134766,
      "logps/rejected": -72.79359436035156,
      "loss": 0.4646216928958893,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3376343548297882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10579778999090195,
      "rewards/margins": 5.431089878082275,
      "rewards/rejected": -5.325291633605957,
      "step": 218,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5507701980509274,
      "grad_norm": 7.270425319671631,
      "learning_rate": 0.00019040449553850257,
      "logits/chosen": -0.4019389748573303,
      "logits/rejected": -0.6102054715156555,
      "logps/chosen": -8.833084106445312,
      "logps/rejected": -68.63448333740234,
      "loss": 0.931887686252594,
      "memory(GiB)": 46.14,
      "nll_loss": 0.6732166409492493,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.29061418771743774,
      "rewards/margins": 4.946336269378662,
      "rewards/rejected": -5.236949920654297,
      "step": 219,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.5532851304621188,
      "grad_norm": 1.4706538915634155,
      "learning_rate": 0.000190285417115109,
      "logits/chosen": -0.44927898049354553,
      "logits/rejected": -0.6523910164833069,
      "logps/chosen": -6.016222953796387,
      "logps/rejected": -71.89757537841797,
      "loss": 0.5478033423423767,
      "memory(GiB)": 46.14,
      "nll_loss": 0.45914456248283386,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15853077173233032,
      "rewards/margins": 5.350086212158203,
      "rewards/rejected": -5.191555500030518,
      "step": 220,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.5558000628733103,
      "grad_norm": 4.954250812530518,
      "learning_rate": 0.00019016564207824992,
      "logits/chosen": -0.49154138565063477,
      "logits/rejected": -0.5963756442070007,
      "logps/chosen": -6.144622325897217,
      "logps/rejected": -47.65474319458008,
      "loss": 0.753817081451416,
      "memory(GiB)": 46.14,
      "nll_loss": 0.5517919659614563,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.045755282044410706,
      "rewards/margins": 3.4072749614715576,
      "rewards/rejected": -3.4530303478240967,
      "step": 221,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.5583149952845017,
      "grad_norm": 20.300432205200195,
      "learning_rate": 0.00019004517135207127,
      "logits/chosen": -0.4070438742637634,
      "logits/rejected": -0.6205348372459412,
      "logps/chosen": -4.287306308746338,
      "logps/rejected": -71.27021789550781,
      "loss": 0.3889833390712738,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2950345575809479,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24340416491031647,
      "rewards/margins": 5.386775493621826,
      "rewards/rejected": -5.143371105194092,
      "step": 222,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.5608299276956932,
      "grad_norm": 4.125121116638184,
      "learning_rate": 0.00018992400586608676,
      "logits/chosen": -0.39273932576179504,
      "logits/rejected": -0.6418203115463257,
      "logps/chosen": -2.4810829162597656,
      "logps/rejected": -68.2148208618164,
      "loss": 0.44946739077568054,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2778445780277252,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.022916745394468307,
      "rewards/margins": 4.892718315124512,
      "rewards/rejected": -4.915634632110596,
      "step": 223,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.5633448601068847,
      "grad_norm": 1.3127343654632568,
      "learning_rate": 0.00018980214655517057,
      "logits/chosen": -0.3633681833744049,
      "logits/rejected": -0.6210101246833801,
      "logps/chosen": -2.5004661083221436,
      "logps/rejected": -63.47230911254883,
      "loss": 0.2738821506500244,
      "memory(GiB)": 46.14,
      "nll_loss": 0.1929439902305603,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16036228835582733,
      "rewards/margins": 4.588406085968018,
      "rewards/rejected": -4.428044319152832,
      "step": 224,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.565859792518076,
      "grad_norm": 1.8078713417053223,
      "learning_rate": 0.00018967959435955026,
      "logits/chosen": -0.401949405670166,
      "logits/rejected": -0.654238224029541,
      "logps/chosen": -2.999825954437256,
      "logps/rejected": -78.48786926269531,
      "loss": 0.30304455757141113,
      "memory(GiB)": 46.14,
      "nll_loss": 0.22698336839675903,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11030103266239166,
      "rewards/margins": 6.056008338928223,
      "rewards/rejected": -5.945706844329834,
      "step": 225,
      "train_speed(iter/s)": 0.011227
    },
    {
      "epoch": 0.5683747249292675,
      "grad_norm": 1.1858668327331543,
      "learning_rate": 0.0001895563502247995,
      "logits/chosen": -0.42564502358436584,
      "logits/rejected": -0.617318868637085,
      "logps/chosen": -4.143329620361328,
      "logps/rejected": -63.9769287109375,
      "loss": 0.4845999479293823,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3030940592288971,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.05418945848941803,
      "rewards/margins": 4.7669267654418945,
      "rewards/rejected": -4.712737083435059,
      "step": 226,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5708896573404589,
      "grad_norm": 10.818222045898438,
      "learning_rate": 0.00018943241510183063,
      "logits/chosen": -0.4244062006473541,
      "logits/rejected": -0.5733806490898132,
      "logps/chosen": -6.869604587554932,
      "logps/rejected": -59.63373565673828,
      "loss": 0.5700305700302124,
      "memory(GiB)": 46.14,
      "nll_loss": 0.42680811882019043,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11073963344097137,
      "rewards/margins": 4.200588226318359,
      "rewards/rejected": -4.089848518371582,
      "step": 227,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5734045897516504,
      "grad_norm": 1.6835912466049194,
      "learning_rate": 0.00018930778994688757,
      "logits/chosen": -0.43102747201919556,
      "logits/rejected": -0.5742402076721191,
      "logps/chosen": -5.771162986755371,
      "logps/rejected": -63.63447952270508,
      "loss": 0.5969828963279724,
      "memory(GiB)": 46.14,
      "nll_loss": 0.43208059668540955,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05310658738017082,
      "rewards/margins": 4.736733436584473,
      "rewards/rejected": -4.683626651763916,
      "step": 228,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5759195221628419,
      "grad_norm": 11.73387336730957,
      "learning_rate": 0.00018918247572153823,
      "logits/chosen": -0.40985164046287537,
      "logits/rejected": -0.5588063597679138,
      "logps/chosen": -4.359460830688477,
      "logps/rejected": -66.97330474853516,
      "loss": 0.4700680375099182,
      "memory(GiB)": 46.14,
      "nll_loss": 0.352486252784729,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08524496108293533,
      "rewards/margins": 5.1315016746521,
      "rewards/rejected": -5.046257019042969,
      "step": 229,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5784344545740333,
      "grad_norm": 3.2223124504089355,
      "learning_rate": 0.0001890564733926672,
      "logits/chosen": -0.28104835748672485,
      "logits/rejected": -0.4876455068588257,
      "logps/chosen": -10.071537017822266,
      "logps/rejected": -77.56631469726562,
      "loss": 0.5610992908477783,
      "memory(GiB)": 46.14,
      "nll_loss": 0.38141241669654846,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.04872121661901474,
      "rewards/margins": 5.269710063934326,
      "rewards/rejected": -5.3184309005737305,
      "step": 230,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5809493869852248,
      "grad_norm": 2.0458524227142334,
      "learning_rate": 0.00018892978393246818,
      "logits/chosen": -0.43532639741897583,
      "logits/rejected": -0.4986593723297119,
      "logps/chosen": -9.826902389526367,
      "logps/rejected": -54.27751541137695,
      "loss": 0.5817813873291016,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3644365966320038,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17297013103961945,
      "rewards/margins": 4.145596981048584,
      "rewards/rejected": -3.972626209259033,
      "step": 231,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5834643193964162,
      "grad_norm": 1.3154174089431763,
      "learning_rate": 0.00018880240831843666,
      "logits/chosen": -0.3312157690525055,
      "logits/rejected": -0.538334310054779,
      "logps/chosen": -7.209171295166016,
      "logps/rejected": -73.7807846069336,
      "loss": 0.4854442775249481,
      "memory(GiB)": 46.14,
      "nll_loss": 0.35912996530532837,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1784784197807312,
      "rewards/margins": 5.071676254272461,
      "rewards/rejected": -4.893198013305664,
      "step": 232,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5859792518076077,
      "grad_norm": 2.2449073791503906,
      "learning_rate": 0.00018867434753336223,
      "logits/chosen": -0.4506746828556061,
      "logits/rejected": -0.5418546795845032,
      "logps/chosen": -6.671032428741455,
      "logps/rejected": -43.13252639770508,
      "loss": 0.6598604917526245,
      "memory(GiB)": 46.14,
      "nll_loss": 0.4920580983161926,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15326479077339172,
      "rewards/margins": 3.1810433864593506,
      "rewards/rejected": -3.0277788639068604,
      "step": 233,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5884941842187991,
      "grad_norm": 2.0205371379852295,
      "learning_rate": 0.000188545602565321,
      "logits/chosen": -0.44379571080207825,
      "logits/rejected": -0.5399007797241211,
      "logps/chosen": -6.668996810913086,
      "logps/rejected": -44.872074127197266,
      "loss": 0.47623366117477417,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3232463002204895,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21571332216262817,
      "rewards/margins": 3.2807154655456543,
      "rewards/rejected": -3.065002202987671,
      "step": 234,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5910091166299906,
      "grad_norm": 2.841298818588257,
      "learning_rate": 0.0001884161744076681,
      "logits/chosen": -0.33942893147468567,
      "logits/rejected": -0.5364092588424683,
      "logps/chosen": -5.104942321777344,
      "logps/rejected": -50.68601989746094,
      "loss": 0.5965795516967773,
      "memory(GiB)": 46.14,
      "nll_loss": 0.4035502076148987,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.029007025063037872,
      "rewards/margins": 3.369378089904785,
      "rewards/rejected": -3.3403711318969727,
      "step": 235,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.5935240490411821,
      "grad_norm": 1.2584537267684937,
      "learning_rate": 0.00018828606405902986,
      "logits/chosen": -0.36284250020980835,
      "logits/rejected": -0.5811104774475098,
      "logps/chosen": -2.7690603733062744,
      "logps/rejected": -57.4227294921875,
      "loss": 0.287708044052124,
      "memory(GiB)": 46.14,
      "nll_loss": 0.19668889045715332,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1330767720937729,
      "rewards/margins": 4.162973880767822,
      "rewards/rejected": -4.029897212982178,
      "step": 236,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.5960389814523734,
      "grad_norm": 1.2746140956878662,
      "learning_rate": 0.00018815527252329624,
      "logits/chosen": -0.43088293075561523,
      "logits/rejected": -0.5826693177223206,
      "logps/chosen": -6.570635795593262,
      "logps/rejected": -57.39180374145508,
      "loss": 0.6008946895599365,
      "memory(GiB)": 46.14,
      "nll_loss": 0.4163006544113159,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07822921127080917,
      "rewards/margins": 4.272268772125244,
      "rewards/rejected": -4.194039821624756,
      "step": 237,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.5985539138635649,
      "grad_norm": 2.0402894020080566,
      "learning_rate": 0.00018802380080961296,
      "logits/chosen": -0.3829416036605835,
      "logits/rejected": -0.6285839676856995,
      "logps/chosen": -2.684177875518799,
      "logps/rejected": -72.15119934082031,
      "loss": 0.38798508048057556,
      "memory(GiB)": 46.14,
      "nll_loss": 0.28602004051208496,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.056711360812187195,
      "rewards/margins": 5.393987655639648,
      "rewards/rejected": -5.337275981903076,
      "step": 238,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.6010688462747563,
      "grad_norm": 1.6787519454956055,
      "learning_rate": 0.00018789164993237382,
      "logits/chosen": -0.4566512107849121,
      "logits/rejected": -0.6156482100486755,
      "logps/chosen": -4.155914783477783,
      "logps/rejected": -73.04944610595703,
      "loss": 0.37050700187683105,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2093135416507721,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13006967306137085,
      "rewards/margins": 5.779219150543213,
      "rewards/rejected": -5.649148941040039,
      "step": 239,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.6035837786859478,
      "grad_norm": 2.6302237510681152,
      "learning_rate": 0.0001877588209112128,
      "logits/chosen": -0.37822332978248596,
      "logits/rejected": -0.6501820087432861,
      "logps/chosen": -3.1933681964874268,
      "logps/rejected": -80.41326904296875,
      "loss": 0.43696874380111694,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2772809863090515,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16505186259746552,
      "rewards/margins": 6.543651103973389,
      "rewards/rejected": -6.378600120544434,
      "step": 240,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.6060987110971393,
      "grad_norm": 1.6916310787200928,
      "learning_rate": 0.00018762531477099626,
      "logits/chosen": -0.29892975091934204,
      "logits/rejected": -0.6198312640190125,
      "logps/chosen": -5.560736656188965,
      "logps/rejected": -102.83201599121094,
      "loss": 0.4650719165802002,
      "memory(GiB)": 46.14,
      "nll_loss": 0.33948570489883423,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.010243753902614117,
      "rewards/margins": 7.854136943817139,
      "rewards/rejected": -7.843893051147461,
      "step": 241,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.6086136435083307,
      "grad_norm": 5.208255290985107,
      "learning_rate": 0.00018749113254181498,
      "logits/chosen": -0.5256325006484985,
      "logits/rejected": -0.7164933681488037,
      "logps/chosen": -2.9781463146209717,
      "logps/rejected": -80.5984878540039,
      "loss": 0.5060344338417053,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3545001149177551,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0441756509244442,
      "rewards/margins": 7.027956485748291,
      "rewards/rejected": -6.983780860900879,
      "step": 242,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6111285759195222,
      "grad_norm": 2.999555826187134,
      "learning_rate": 0.0001873562752589762,
      "logits/chosen": -0.4497723877429962,
      "logits/rejected": -0.6424503326416016,
      "logps/chosen": -9.136008262634277,
      "logps/rejected": -83.97636413574219,
      "loss": 0.5317915678024292,
      "memory(GiB)": 46.14,
      "nll_loss": 0.45959895849227905,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -0.0020806342363357544,
      "rewards/margins": 6.646152496337891,
      "rewards/rejected": -6.648233890533447,
      "step": 243,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6136435083307136,
      "grad_norm": 1.6152504682540894,
      "learning_rate": 0.00018722074396299566,
      "logits/chosen": -0.4504960775375366,
      "logits/rejected": -0.6728327870368958,
      "logps/chosen": -8.307985305786133,
      "logps/rejected": -92.50747680664062,
      "loss": 0.3881627023220062,
      "memory(GiB)": 46.14,
      "nll_loss": 0.27829721570014954,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1872478723526001,
      "rewards/margins": 7.5645575523376465,
      "rewards/rejected": -7.377309799194336,
      "step": 244,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6161584407419051,
      "grad_norm": 1.9773950576782227,
      "learning_rate": 0.00018708453969958958,
      "logits/chosen": -0.48630112409591675,
      "logits/rejected": -0.6566104888916016,
      "logps/chosen": -6.018984317779541,
      "logps/rejected": -62.95851516723633,
      "loss": 0.6716858744621277,
      "memory(GiB)": 46.14,
      "nll_loss": 0.48807603120803833,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0030519720166921616,
      "rewards/margins": 4.69732141494751,
      "rewards/rejected": -4.694269180297852,
      "step": 245,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6186733731530966,
      "grad_norm": 2.007908821105957,
      "learning_rate": 0.00018694766351966667,
      "logits/chosen": -0.4003494679927826,
      "logits/rejected": -0.6231962442398071,
      "logps/chosen": -6.337024688720703,
      "logps/rejected": -75.96829223632812,
      "loss": 0.49646520614624023,
      "memory(GiB)": 46.14,
      "nll_loss": 0.39788398146629333,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2538176476955414,
      "rewards/margins": 5.768869876861572,
      "rewards/rejected": -5.515052318572998,
      "step": 246,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.621188305564288,
      "grad_norm": 1.3329335451126099,
      "learning_rate": 0.00018681011647931974,
      "logits/chosen": -0.35181480646133423,
      "logits/rejected": -0.6176152229309082,
      "logps/chosen": -5.239382266998291,
      "logps/rejected": -74.54182434082031,
      "loss": 0.5128018856048584,
      "memory(GiB)": 46.14,
      "nll_loss": 0.3926093578338623,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14708997309207916,
      "rewards/margins": 5.654610633850098,
      "rewards/rejected": -5.5075201988220215,
      "step": 247,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6237032379754794,
      "grad_norm": 2.318981170654297,
      "learning_rate": 0.00018667189963981793,
      "logits/chosen": -0.4104291498661041,
      "logits/rejected": -0.6699872016906738,
      "logps/chosen": -4.179399013519287,
      "logps/rejected": -62.75163269042969,
      "loss": 0.595816969871521,
      "memory(GiB)": 46.14,
      "nll_loss": 0.4384332001209259,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17818482220172882,
      "rewards/margins": 4.639157772064209,
      "rewards/rejected": -4.460972309112549,
      "step": 248,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6262181703866708,
      "grad_norm": 1.3054991960525513,
      "learning_rate": 0.00018653301406759825,
      "logits/chosen": -0.45266249775886536,
      "logits/rejected": -0.7092356085777283,
      "logps/chosen": -4.101914882659912,
      "logps/rejected": -68.14440155029297,
      "loss": 0.38595661520957947,
      "memory(GiB)": 46.14,
      "nll_loss": 0.2959481179714203,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18340018391609192,
      "rewards/margins": 5.416258335113525,
      "rewards/rejected": -5.232857704162598,
      "step": 249,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.6287331027978623,
      "grad_norm": 2.184175968170166,
      "learning_rate": 0.00018639346083425748,
      "logits/chosen": -0.5093356370925903,
      "logits/rejected": -0.6794995665550232,
      "logps/chosen": -8.688965797424316,
      "logps/rejected": -66.35594940185547,
      "loss": 0.8350189328193665,
      "memory(GiB)": 46.14,
      "nll_loss": 0.6493321657180786,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0820498839020729,
      "rewards/margins": 5.020396709442139,
      "rewards/rejected": -4.9383463859558105,
      "step": 250,
      "train_speed(iter/s)": 0.011231
    }
  ],
  "logging_steps": 1,
  "max_steps": 1191,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.370929738243113e+17,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}