{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.25149324111914495,
  "eval_steps": 300,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.002514932411191449,
      "grad_norm": 5.559185028076172,
      "learning_rate": 3.3333333333333333e-06,
      "logits/chosen": -0.5158984065055847,
      "logits/rejected": -0.6433685421943665,
      "logps/chosen": -8.879493713378906,
      "logps/rejected": -18.298219680786133,
      "loss": 1.2430726289749146,
      "memory(GiB)": 30.69,
      "nll_loss": 0.5499253869056702,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.010477
    },
    {
      "epoch": 0.005029864822382898,
      "grad_norm": 5.400235176086426,
      "learning_rate": 6.666666666666667e-06,
      "logits/chosen": -0.5219721794128418,
      "logits/rejected": -0.6263731718063354,
      "logps/chosen": -4.77984094619751,
      "logps/rejected": -15.150838851928711,
      "loss": 1.0675525665283203,
      "memory(GiB)": 33.9,
      "nll_loss": 0.3744054436683655,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.010866
    },
    {
      "epoch": 0.007544797233574348,
      "grad_norm": 7.852550983428955,
      "learning_rate": 1e-05,
      "logits/chosen": -0.48881009221076965,
      "logits/rejected": -0.6540625691413879,
      "logps/chosen": -4.714770793914795,
      "logps/rejected": -17.99374008178711,
      "loss": 1.0851404666900635,
      "memory(GiB)": 33.9,
      "nll_loss": 0.3894187808036804,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.002094469266012311,
      "rewards/margins": -0.004971538204699755,
      "rewards/rejected": 0.0028770684730261564,
      "step": 3,
      "train_speed(iter/s)": 0.011002
    },
    {
      "epoch": 0.010059729644765796,
      "grad_norm": 10.150762557983398,
      "learning_rate": 1.3333333333333333e-05,
      "logits/chosen": -0.4506370425224304,
      "logits/rejected": -0.6422093510627747,
      "logps/chosen": -3.6849846839904785,
      "logps/rejected": -21.201997756958008,
      "loss": 1.092491865158081,
      "memory(GiB)": 37.36,
      "nll_loss": 0.4020901322364807,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.007022961974143982,
      "rewards/margins": 0.0057290042750537395,
      "rewards/rejected": 0.001293957349844277,
      "step": 4,
      "train_speed(iter/s)": 0.011051
    },
    {
      "epoch": 0.012574662055957246,
      "grad_norm": 6.588510036468506,
      "learning_rate": 1.6666666666666667e-05,
      "logits/chosen": -0.4159216284751892,
      "logits/rejected": -0.5884239673614502,
      "logps/chosen": -8.611150741577148,
      "logps/rejected": -19.931533813476562,
      "loss": 1.1986628770828247,
      "memory(GiB)": 37.36,
      "nll_loss": 0.5155895948410034,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.005048489198088646,
      "rewards/margins": 0.020845137536525726,
      "rewards/rejected": -0.01579664833843708,
      "step": 5,
      "train_speed(iter/s)": 0.011086
    },
    {
      "epoch": 0.015089594467148696,
      "grad_norm": 3.965754508972168,
      "learning_rate": 2e-05,
      "logits/chosen": -0.47118157148361206,
      "logits/rejected": -0.5790206789970398,
      "logps/chosen": -9.572708129882812,
      "logps/rejected": -19.066829681396484,
      "loss": 1.1965497732162476,
      "memory(GiB)": 37.36,
      "nll_loss": 0.5279884338378906,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.006116065196692944,
      "rewards/margins": 0.05174415558576584,
      "rewards/rejected": -0.04562808945775032,
      "step": 6,
      "train_speed(iter/s)": 0.01111
    },
    {
      "epoch": 0.017604526878340146,
      "grad_norm": 2.719625234603882,
      "learning_rate": 2.3333333333333336e-05,
      "logits/chosen": -0.40740644931793213,
      "logits/rejected": -0.5419386029243469,
      "logps/chosen": -9.413152694702148,
      "logps/rejected": -24.703401565551758,
      "loss": 1.1549676656723022,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5149396061897278,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.02250390499830246,
      "rewards/margins": 0.12123934924602509,
      "rewards/rejected": -0.14374326169490814,
      "step": 7,
      "train_speed(iter/s)": 0.011114
    },
    {
      "epoch": 0.020119459289531592,
      "grad_norm": 3.325852394104004,
      "learning_rate": 2.6666666666666667e-05,
      "logits/chosen": -0.4091758728027344,
      "logits/rejected": -0.5692812204360962,
      "logps/chosen": -5.904629230499268,
      "logps/rejected": -20.752662658691406,
      "loss": 1.0213468074798584,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3945533037185669,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.041384391486644745,
      "rewards/margins": 0.17802417278289795,
      "rewards/rejected": -0.2194085717201233,
      "step": 8,
      "train_speed(iter/s)": 0.011127
    },
    {
      "epoch": 0.022634391700723042,
      "grad_norm": 3.5531411170959473,
      "learning_rate": 3e-05,
      "logits/chosen": -0.3281947374343872,
      "logits/rejected": -0.5156837701797485,
      "logps/chosen": -5.975313186645508,
      "logps/rejected": -27.066158294677734,
      "loss": 0.8695712089538574,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3611774742603302,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0457351878285408,
      "rewards/margins": 0.4895857870578766,
      "rewards/rejected": -0.443850576877594,
      "step": 9,
      "train_speed(iter/s)": 0.011136
    },
    {
      "epoch": 0.025149324111914492,
      "grad_norm": 4.356088161468506,
      "learning_rate": 3.3333333333333335e-05,
      "logits/chosen": -0.43971675634384155,
      "logits/rejected": -0.5702978372573853,
      "logps/chosen": -6.376318454742432,
      "logps/rejected": -19.42030143737793,
      "loss": 1.0046417713165283,
      "memory(GiB)": 40.85,
      "nll_loss": 0.42133060097694397,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.17984721064567566,
      "rewards/margins": 0.43266722559928894,
      "rewards/rejected": -0.6125144362449646,
      "step": 10,
      "train_speed(iter/s)": 0.01115
    },
    {
      "epoch": 0.027664256523105942,
      "grad_norm": 3.9732213020324707,
      "learning_rate": 3.6666666666666666e-05,
      "logits/chosen": -0.4561832845211029,
      "logits/rejected": -0.6065505146980286,
      "logps/chosen": -6.498957633972168,
      "logps/rejected": -27.428552627563477,
      "loss": 0.9565106630325317,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40273764729499817,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.12391924858093262,
      "rewards/margins": 0.5706068277359009,
      "rewards/rejected": -0.6945260763168335,
      "step": 11,
      "train_speed(iter/s)": 0.011161
    },
    {
      "epoch": 0.030179188934297392,
      "grad_norm": 2.2540578842163086,
      "learning_rate": 4e-05,
      "logits/chosen": -0.42088037729263306,
      "logits/rejected": -0.6009025573730469,
      "logps/chosen": -4.243446350097656,
      "logps/rejected": -26.567686080932617,
      "loss": 0.7599425911903381,
      "memory(GiB)": 40.85,
      "nll_loss": 0.28007903695106506,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.14110325276851654,
      "rewards/margins": 0.6590132713317871,
      "rewards/rejected": -0.8001165390014648,
      "step": 12,
      "train_speed(iter/s)": 0.011171
    },
    {
      "epoch": 0.03269412134548884,
      "grad_norm": 6.460833549499512,
      "learning_rate": 4.3333333333333334e-05,
      "logits/chosen": -0.4136754274368286,
      "logits/rejected": -0.5452494621276855,
      "logps/chosen": -7.585506439208984,
      "logps/rejected": -21.19340705871582,
      "loss": 1.223608136177063,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6720148921012878,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.25483113527297974,
      "rewards/margins": 0.49035438895225525,
      "rewards/rejected": -0.7451854944229126,
      "step": 13,
      "train_speed(iter/s)": 0.011181
    },
    {
      "epoch": 0.03520905375668029,
      "grad_norm": 4.092947006225586,
      "learning_rate": 4.666666666666667e-05,
      "logits/chosen": -0.527854323387146,
      "logits/rejected": -0.5994939804077148,
      "logps/chosen": -8.185604095458984,
      "logps/rejected": -17.946279525756836,
      "loss": 1.2557107210159302,
      "memory(GiB)": 40.85,
      "nll_loss": 0.66472989320755,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.145107239484787,
      "rewards/margins": 0.41058170795440674,
      "rewards/rejected": -0.5556889772415161,
      "step": 14,
      "train_speed(iter/s)": 0.011194
    },
    {
      "epoch": 0.03772398616787174,
      "grad_norm": 3.926342248916626,
      "learning_rate": 5e-05,
      "logits/chosen": -0.407508909702301,
      "logits/rejected": -0.5949459075927734,
      "logps/chosen": -3.4148104190826416,
      "logps/rejected": -22.69104766845703,
      "loss": 0.8548452258110046,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40506720542907715,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.025914989411830902,
      "rewards/margins": 0.6952927708625793,
      "rewards/rejected": -0.7212077379226685,
      "step": 15,
      "train_speed(iter/s)": 0.0112
    },
    {
      "epoch": 0.040238918579063185,
      "grad_norm": 2.4273853302001953,
      "learning_rate": 5.333333333333333e-05,
      "logits/chosen": -0.3462049961090088,
      "logits/rejected": -0.58342045545578,
      "logps/chosen": -3.8651039600372314,
      "logps/rejected": -28.470134735107422,
      "loss": 0.821665346622467,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3309793472290039,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04178892821073532,
      "rewards/margins": 0.5164157152175903,
      "rewards/rejected": -0.4746267795562744,
      "step": 16,
      "train_speed(iter/s)": 0.011202
    },
    {
      "epoch": 0.04275385099025464,
      "grad_norm": 1.8921568393707275,
      "learning_rate": 5.666666666666667e-05,
      "logits/chosen": -0.5054872632026672,
      "logits/rejected": -0.6153637170791626,
      "logps/chosen": -1.8910857439041138,
      "logps/rejected": -16.781475067138672,
      "loss": 0.7665270566940308,
      "memory(GiB)": 40.85,
      "nll_loss": 0.2412014603614807,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04062865674495697,
      "rewards/margins": 0.41468262672424316,
      "rewards/rejected": -0.3740540146827698,
      "step": 17,
      "train_speed(iter/s)": 0.011212
    },
    {
      "epoch": 0.045268783401446085,
      "grad_norm": 3.2724339962005615,
      "learning_rate": 6e-05,
      "logits/chosen": -0.45994284749031067,
      "logits/rejected": -0.5327339768409729,
      "logps/chosen": -8.705971717834473,
      "logps/rejected": -17.33376121520996,
      "loss": 1.2050995826721191,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5643416047096252,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.048418305814266205,
      "rewards/margins": 0.16582812368869781,
      "rewards/rejected": -0.21424642205238342,
      "step": 18,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.04778371581263754,
      "grad_norm": 2.6650567054748535,
      "learning_rate": 6.333333333333333e-05,
      "logits/chosen": -0.48406994342803955,
      "logits/rejected": -0.5535129308700562,
      "logps/chosen": -10.087296485900879,
      "logps/rejected": -17.584102630615234,
      "loss": 1.1602699756622314,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5509804487228394,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005405411124229431,
      "rewards/margins": 0.23162491619586945,
      "rewards/rejected": -0.23703034222126007,
      "step": 19,
      "train_speed(iter/s)": 0.01122
    },
    {
      "epoch": 0.050298648223828984,
      "grad_norm": 2.393120527267456,
      "learning_rate": 6.666666666666667e-05,
      "logits/chosen": -0.4699333906173706,
      "logits/rejected": -0.5745525360107422,
      "logps/chosen": -5.482749938964844,
      "logps/rejected": -15.294036865234375,
      "loss": 1.0463979244232178,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40886539220809937,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.030414806678891182,
      "rewards/margins": 0.18470264971256256,
      "rewards/rejected": -0.15428784489631653,
      "step": 20,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.05281358063502043,
      "grad_norm": 1.7328065633773804,
      "learning_rate": 7e-05,
      "logits/chosen": -0.5130109190940857,
      "logits/rejected": -0.6171243190765381,
      "logps/chosen": -8.619268417358398,
      "logps/rejected": -18.43971824645996,
      "loss": 1.1801689863204956,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6120539903640747,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07324957102537155,
      "rewards/margins": 0.3033149242401123,
      "rewards/rejected": -0.23006536066532135,
      "step": 21,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.055328513046211884,
      "grad_norm": 2.848578691482544,
      "learning_rate": 7.333333333333333e-05,
      "logits/chosen": -0.46683233976364136,
      "logits/rejected": -0.5329924821853638,
      "logps/chosen": -4.357663154602051,
      "logps/rejected": -15.35399055480957,
      "loss": 0.9406945705413818,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3216635584831238,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.010179778560996056,
      "rewards/margins": 0.2008945494890213,
      "rewards/rejected": -0.21107429265975952,
      "step": 22,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.05784344545740333,
      "grad_norm": 1.7426271438598633,
      "learning_rate": 7.666666666666667e-05,
      "logits/chosen": -0.5002316236495972,
      "logits/rejected": -0.5488481521606445,
      "logps/chosen": -5.690494537353516,
      "logps/rejected": -15.590461730957031,
      "loss": 0.9775924682617188,
      "memory(GiB)": 40.85,
      "nll_loss": 0.355554461479187,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.009325886145234108,
      "rewards/margins": 0.18430516123771667,
      "rewards/rejected": -0.1749793142080307,
      "step": 23,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.060358377868594784,
      "grad_norm": 1.8518409729003906,
      "learning_rate": 8e-05,
      "logits/chosen": -0.5258264541625977,
      "logits/rejected": -0.5805689096450806,
      "logps/chosen": -9.133979797363281,
      "logps/rejected": -17.886051177978516,
      "loss": 1.1810051202774048,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6029603481292725,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.04269062727689743,
      "rewards/margins": 0.2787190079689026,
      "rewards/rejected": -0.23602835834026337,
      "step": 24,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.06287331027978624,
      "grad_norm": 2.4842066764831543,
      "learning_rate": 8.333333333333334e-05,
      "logits/chosen": -0.44220152497291565,
      "logits/rejected": -0.5788168907165527,
      "logps/chosen": -4.989194869995117,
      "logps/rejected": -18.047056198120117,
      "loss": 1.0011777877807617,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4066605269908905,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.021916719153523445,
      "rewards/margins": 0.2587287425994873,
      "rewards/rejected": -0.236812025308609,
      "step": 25,
      "train_speed(iter/s)": 0.011239
    },
    {
      "epoch": 0.06538824269097768,
      "grad_norm": 1.766322135925293,
      "learning_rate": 8.666666666666667e-05,
      "logits/chosen": -0.3389984369277954,
      "logits/rejected": -0.543292224407196,
      "logps/chosen": -11.735610961914062,
      "logps/rejected": -26.25033950805664,
      "loss": 1.0063844919204712,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5172990560531616,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07315510511398315,
      "rewards/margins": 0.5734083652496338,
      "rewards/rejected": -0.5002533197402954,
      "step": 26,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.06790317510216913,
      "grad_norm": 3.3314130306243896,
      "learning_rate": 9e-05,
      "logits/chosen": -0.37741950154304504,
      "logits/rejected": -0.5884826183319092,
      "logps/chosen": -5.442512035369873,
      "logps/rejected": -27.738323211669922,
      "loss": 0.8821365237236023,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4055330753326416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.03083166666328907,
      "rewards/margins": 0.6056570410728455,
      "rewards/rejected": -0.6364887952804565,
      "step": 27,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.07041810751336058,
      "grad_norm": 2.849647283554077,
      "learning_rate": 9.333333333333334e-05,
      "logits/chosen": -0.4310716688632965,
      "logits/rejected": -0.5529810190200806,
      "logps/chosen": -8.224971771240234,
      "logps/rejected": -26.79846954345703,
      "loss": 0.8747767806053162,
      "memory(GiB)": 40.85,
      "nll_loss": 0.45313140749931335,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.07478700578212738,
      "rewards/margins": 0.8052245378494263,
      "rewards/rejected": -0.8800115585327148,
      "step": 28,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.07293303992455202,
      "grad_norm": 2.247368097305298,
      "learning_rate": 9.666666666666667e-05,
      "logits/chosen": -0.37296879291534424,
      "logits/rejected": -0.5259015560150146,
      "logps/chosen": -6.546780109405518,
      "logps/rejected": -25.603477478027344,
      "loss": 0.9804578423500061,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4873877465724945,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.046319298446178436,
      "rewards/margins": 0.5485984086990356,
      "rewards/rejected": -0.5949177145957947,
      "step": 29,
      "train_speed(iter/s)": 0.011239
    },
    {
      "epoch": 0.07544797233574348,
      "grad_norm": 1.7451585531234741,
      "learning_rate": 0.0001,
      "logits/chosen": -0.36992424726486206,
      "logits/rejected": -0.45540890097618103,
      "logps/chosen": -6.483048439025879,
      "logps/rejected": -18.744525909423828,
      "loss": 0.9453181624412537,
      "memory(GiB)": 40.85,
      "nll_loss": 0.37866073846817017,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0520947091281414,
      "rewards/margins": 0.33587026596069336,
      "rewards/rejected": -0.28377556800842285,
      "step": 30,
      "train_speed(iter/s)": 0.01124
    },
    {
      "epoch": 0.07796290474693493,
      "grad_norm": 2.8843863010406494,
      "learning_rate": 0.00010333333333333334,
      "logits/chosen": -0.3973531723022461,
      "logits/rejected": -0.4880366325378418,
      "logps/chosen": -6.5046491622924805,
      "logps/rejected": -18.793209075927734,
      "loss": 1.0352380275726318,
      "memory(GiB)": 40.85,
      "nll_loss": 0.514582097530365,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06333249807357788,
      "rewards/margins": 0.5268359780311584,
      "rewards/rejected": -0.4635034203529358,
      "step": 31,
      "train_speed(iter/s)": 0.011242
    },
    {
      "epoch": 0.08047783715812637,
      "grad_norm": 1.9934616088867188,
      "learning_rate": 0.00010666666666666667,
      "logits/chosen": -0.38435813784599304,
      "logits/rejected": -0.4866703450679779,
      "logps/chosen": -5.984093189239502,
      "logps/rejected": -28.37721061706543,
      "loss": 0.9228850603103638,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3919341564178467,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.08762034773826599,
      "rewards/margins": 0.4990919232368469,
      "rewards/rejected": -0.41147157549858093,
      "step": 32,
      "train_speed(iter/s)": 0.011242
    },
    {
      "epoch": 0.08299276956931782,
      "grad_norm": 2.605156421661377,
      "learning_rate": 0.00011000000000000002,
      "logits/chosen": -0.3683161437511444,
      "logits/rejected": -0.4800977110862732,
      "logps/chosen": -6.79887056350708,
      "logps/rejected": -18.521530151367188,
      "loss": 0.9287618398666382,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4155515730381012,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.062238909304142,
      "rewards/margins": 0.4814634323120117,
      "rewards/rejected": -0.41922450065612793,
      "step": 33,
      "train_speed(iter/s)": 0.011243
    },
    {
      "epoch": 0.08550770198050928,
      "grad_norm": 2.1593198776245117,
      "learning_rate": 0.00011333333333333334,
      "logits/chosen": -0.3462904989719391,
      "logits/rejected": -0.4482634663581848,
      "logps/chosen": -4.484028339385986,
      "logps/rejected": -22.155223846435547,
      "loss": 0.8287199139595032,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3564324378967285,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.011370105668902397,
      "rewards/margins": 0.8015316724777222,
      "rewards/rejected": -0.8129018545150757,
      "step": 34,
      "train_speed(iter/s)": 0.011244
    },
    {
      "epoch": 0.08802263439170073,
      "grad_norm": 2.1223862171173096,
      "learning_rate": 0.00011666666666666668,
      "logits/chosen": -0.23746490478515625,
      "logits/rejected": -0.4123544991016388,
      "logps/chosen": -4.062009334564209,
      "logps/rejected": -30.214771270751953,
      "loss": 0.8381334543228149,
      "memory(GiB)": 40.85,
      "nll_loss": 0.45768678188323975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06588438153266907,
      "rewards/margins": 1.1650195121765137,
      "rewards/rejected": -1.0991352796554565,
      "step": 35,
      "train_speed(iter/s)": 0.011244
    },
    {
      "epoch": 0.09053756680289217,
      "grad_norm": 1.7519793510437012,
      "learning_rate": 0.00012,
      "logits/chosen": -0.274143248796463,
      "logits/rejected": -0.4072067439556122,
      "logps/chosen": -7.12678861618042,
      "logps/rejected": -29.586929321289062,
      "loss": 0.7818687558174133,
      "memory(GiB)": 40.85,
      "nll_loss": 0.38336753845214844,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.030900314450263977,
      "rewards/margins": 1.0601840019226074,
      "rewards/rejected": -1.0292836427688599,
      "step": 36,
      "train_speed(iter/s)": 0.011244
    },
    {
      "epoch": 0.09305249921408362,
      "grad_norm": 2.2621781826019287,
      "learning_rate": 0.00012333333333333334,
      "logits/chosen": -0.30612003803253174,
      "logits/rejected": -0.44930776953697205,
      "logps/chosen": -5.637537956237793,
      "logps/rejected": -30.86564064025879,
      "loss": 0.7518173456192017,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4073091149330139,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1094760149717331,
      "rewards/margins": 1.2509604692459106,
      "rewards/rejected": -1.3604364395141602,
      "step": 37,
      "train_speed(iter/s)": 0.011245
    },
    {
      "epoch": 0.09556743162527508,
      "grad_norm": 2.6005046367645264,
      "learning_rate": 0.00012666666666666666,
      "logits/chosen": -0.36841830611228943,
      "logits/rejected": -0.44501256942749023,
      "logps/chosen": -8.786764144897461,
      "logps/rejected": -21.693777084350586,
      "loss": 0.967311441898346,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5350124835968018,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.05166015028953552,
      "rewards/margins": 0.9184513092041016,
      "rewards/rejected": -0.9701113700866699,
      "step": 38,
      "train_speed(iter/s)": 0.011247
    },
    {
      "epoch": 0.09808236403646652,
      "grad_norm": 7.679723262786865,
      "learning_rate": 0.00013000000000000002,
      "logits/chosen": -0.3933795094490051,
      "logits/rejected": -0.4303474426269531,
      "logps/chosen": -9.630545616149902,
      "logps/rejected": -25.0108642578125,
      "loss": 1.1940371990203857,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6422770023345947,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.19723013043403625,
      "rewards/margins": 0.8150386214256287,
      "rewards/rejected": -1.0122687816619873,
      "step": 39,
      "train_speed(iter/s)": 0.011247
    },
    {
      "epoch": 0.10059729644765797,
      "grad_norm": 2.497746229171753,
      "learning_rate": 0.00013333333333333334,
      "logits/chosen": -0.3561173677444458,
      "logits/rejected": -0.4964173436164856,
      "logps/chosen": -6.288295269012451,
      "logps/rejected": -28.722339630126953,
      "loss": 0.7734813690185547,
      "memory(GiB)": 40.85,
      "nll_loss": 0.35979557037353516,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.020053118467330933,
      "rewards/margins": 1.0713183879852295,
      "rewards/rejected": -1.0512654781341553,
      "step": 40,
      "train_speed(iter/s)": 0.011248
    },
    {
      "epoch": 0.10311222885884942,
      "grad_norm": 3.900786876678467,
      "learning_rate": 0.00013666666666666666,
      "logits/chosen": -0.30438148975372314,
      "logits/rejected": -0.4660162031650543,
      "logps/chosen": -9.80126953125,
      "logps/rejected": -22.89883804321289,
      "loss": 1.0617499351501465,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5348615646362305,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.037312038242816925,
      "rewards/margins": 0.48836010694503784,
      "rewards/rejected": -0.4510480761528015,
      "step": 41,
      "train_speed(iter/s)": 0.011248
    },
    {
      "epoch": 0.10562716127004086,
      "grad_norm": 4.212996006011963,
      "learning_rate": 0.00014,
      "logits/chosen": -0.37392380833625793,
      "logits/rejected": -0.4811669588088989,
      "logps/chosen": -8.54138469696045,
      "logps/rejected": -24.88803482055664,
      "loss": 0.9449828267097473,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40984830260276794,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.078499935567379,
      "rewards/margins": 0.6149317026138306,
      "rewards/rejected": -0.6934316754341125,
      "step": 42,
      "train_speed(iter/s)": 0.011249
    },
    {
      "epoch": 0.10814209368123232,
      "grad_norm": 2.7297940254211426,
      "learning_rate": 0.00014333333333333334,
      "logits/chosen": -0.22235050797462463,
      "logits/rejected": -0.44656193256378174,
      "logps/chosen": -3.8628368377685547,
      "logps/rejected": -29.070890426635742,
      "loss": 0.6864454746246338,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3404179513454437,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10101112723350525,
      "rewards/margins": 1.1784940958023071,
      "rewards/rejected": -1.0774829387664795,
      "step": 43,
      "train_speed(iter/s)": 0.011247
    },
    {
      "epoch": 0.11065702609242377,
      "grad_norm": 4.357929229736328,
      "learning_rate": 0.00014666666666666666,
      "logits/chosen": -0.3152369558811188,
      "logits/rejected": -0.4675106406211853,
      "logps/chosen": -8.47473430633545,
      "logps/rejected": -35.49666976928711,
      "loss": 0.8841890096664429,
      "memory(GiB)": 40.85,
      "nll_loss": 0.543793261051178,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.2599925100803375,
      "rewards/margins": 1.5472975969314575,
      "rewards/rejected": -1.8072898387908936,
      "step": 44,
      "train_speed(iter/s)": 0.011248
    },
    {
      "epoch": 0.11317195850361522,
      "grad_norm": 4.053813934326172,
      "learning_rate": 0.00015000000000000001,
      "logits/chosen": -0.47932106256484985,
      "logits/rejected": -0.5357992053031921,
      "logps/chosen": -10.139131546020508,
      "logps/rejected": -29.30261993408203,
      "loss": 1.0090879201889038,
      "memory(GiB)": 40.85,
      "nll_loss": 0.6002873182296753,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.2646394371986389,
      "rewards/margins": 1.3857942819595337,
      "rewards/rejected": -1.6504336595535278,
      "step": 45,
      "train_speed(iter/s)": 0.01125
    },
    {
      "epoch": 0.11568689091480666,
      "grad_norm": 3.4071342945098877,
      "learning_rate": 0.00015333333333333334,
      "logits/chosen": -0.35965389013290405,
      "logits/rejected": -0.4886631965637207,
      "logps/chosen": -6.492125988006592,
      "logps/rejected": -33.90357971191406,
      "loss": 0.9477238655090332,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5519906878471375,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13393068313598633,
      "rewards/margins": 1.3498430252075195,
      "rewards/rejected": -1.4837737083435059,
      "step": 46,
      "train_speed(iter/s)": 0.011251
    },
    {
      "epoch": 0.11820182332599811,
      "grad_norm": 1.5676898956298828,
      "learning_rate": 0.00015666666666666666,
      "logits/chosen": -0.3339039087295532,
      "logits/rejected": -0.5405474305152893,
      "logps/chosen": -5.18754768371582,
      "logps/rejected": -34.61940383911133,
      "loss": 0.5877978205680847,
      "memory(GiB)": 40.85,
      "nll_loss": 0.26233971118927,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04858784377574921,
      "rewards/margins": 1.4838428497314453,
      "rewards/rejected": -1.4352549314498901,
      "step": 47,
      "train_speed(iter/s)": 0.011252
    },
    {
      "epoch": 0.12071675573718957,
      "grad_norm": 1.7600055932998657,
      "learning_rate": 0.00016,
      "logits/chosen": -0.38554954528808594,
      "logits/rejected": -0.5349220633506775,
      "logps/chosen": -5.35024356842041,
      "logps/rejected": -30.936561584472656,
      "loss": 0.7445188760757446,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3484676480293274,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.04428477957844734,
      "rewards/margins": 1.158116340637207,
      "rewards/rejected": -1.2024011611938477,
      "step": 48,
      "train_speed(iter/s)": 0.011253
    },
    {
      "epoch": 0.12323168814838101,
      "grad_norm": 1.8286348581314087,
      "learning_rate": 0.00016333333333333334,
      "logits/chosen": -0.3060781955718994,
      "logits/rejected": -0.5333737134933472,
      "logps/chosen": -4.790343761444092,
      "logps/rejected": -32.540409088134766,
      "loss": 0.6686277985572815,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3360018730163574,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0967080146074295,
      "rewards/margins": 1.285821795463562,
      "rewards/rejected": -1.1891138553619385,
      "step": 49,
      "train_speed(iter/s)": 0.011254
    },
    {
      "epoch": 0.12574662055957248,
      "grad_norm": 1.8983184099197388,
      "learning_rate": 0.0001666666666666667,
      "logits/chosen": -0.22582519054412842,
      "logits/rejected": -0.47035256028175354,
      "logps/chosen": -6.48522424697876,
      "logps/rejected": -32.73487091064453,
      "loss": 0.8076327443122864,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3926582336425781,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.01478707417845726,
      "rewards/margins": 1.167083501815796,
      "rewards/rejected": -1.1522964239120483,
      "step": 50,
      "train_speed(iter/s)": 0.011253
    },
    {
      "epoch": 0.12826155297076391,
      "grad_norm": 2.4192955493927,
      "learning_rate": 0.00017,
      "logits/chosen": -0.30057138204574585,
      "logits/rejected": -0.4678650498390198,
      "logps/chosen": -7.537961483001709,
      "logps/rejected": -34.26948547363281,
      "loss": 0.8188683390617371,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4251425266265869,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.051345158368349075,
      "rewards/margins": 1.1266952753067017,
      "rewards/rejected": -1.178040623664856,
      "step": 51,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.13077648538195535,
      "grad_norm": 2.0905075073242188,
      "learning_rate": 0.00017333333333333334,
      "logits/chosen": -0.2889178395271301,
      "logits/rejected": -0.49443909525871277,
      "logps/chosen": -6.827200412750244,
      "logps/rejected": -44.23398971557617,
      "loss": 0.7206944227218628,
      "memory(GiB)": 40.85,
      "nll_loss": 0.39006102085113525,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06844954192638397,
      "rewards/margins": 2.328122854232788,
      "rewards/rejected": -2.3965721130371094,
      "step": 52,
      "train_speed(iter/s)": 0.01121
    },
    {
      "epoch": 0.13329141779314682,
      "grad_norm": 1.9464434385299683,
      "learning_rate": 0.00017666666666666666,
      "logits/chosen": -0.30524271726608276,
      "logits/rejected": -0.5259577631950378,
      "logps/chosen": -8.820595741271973,
      "logps/rejected": -34.23869705200195,
      "loss": 0.8924205899238586,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5174475312232971,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.007829657755792141,
      "rewards/margins": 1.6594951152801514,
      "rewards/rejected": -1.6673249006271362,
      "step": 53,
      "train_speed(iter/s)": 0.011211
    },
    {
      "epoch": 0.13580635020433826,
      "grad_norm": 2.15484356880188,
      "learning_rate": 0.00018,
      "logits/chosen": -0.36469364166259766,
      "logits/rejected": -0.5003997087478638,
      "logps/chosen": -9.828875541687012,
      "logps/rejected": -35.949886322021484,
      "loss": 0.8116447925567627,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4780670404434204,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04445749148726463,
      "rewards/margins": 1.7651525735855103,
      "rewards/rejected": -1.7206950187683105,
      "step": 54,
      "train_speed(iter/s)": 0.011212
    },
    {
      "epoch": 0.1383212826155297,
      "grad_norm": 5.288140773773193,
      "learning_rate": 0.00018333333333333334,
      "logits/chosen": -0.357359915971756,
      "logits/rejected": -0.5485042333602905,
      "logps/chosen": -4.562381744384766,
      "logps/rejected": -40.544979095458984,
      "loss": 0.8048219680786133,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4945175051689148,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.026466388255357742,
      "rewards/margins": 2.5180537700653076,
      "rewards/rejected": -2.544520378112793,
      "step": 55,
      "train_speed(iter/s)": 0.011214
    },
    {
      "epoch": 0.14083621502672117,
      "grad_norm": 4.063258647918701,
      "learning_rate": 0.0001866666666666667,
      "logits/chosen": -0.39062973856925964,
      "logits/rejected": -0.5040320158004761,
      "logps/chosen": -10.983360290527344,
      "logps/rejected": -33.32127380371094,
      "loss": 1.1587333679199219,
      "memory(GiB)": 40.85,
      "nll_loss": 0.7283708453178406,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.13727927207946777,
      "rewards/margins": 1.9422543048858643,
      "rewards/rejected": -2.079533576965332,
      "step": 56,
      "train_speed(iter/s)": 0.011215
    },
    {
      "epoch": 0.1433511474379126,
      "grad_norm": 3.7057104110717773,
      "learning_rate": 0.00019,
      "logits/chosen": -0.34116944670677185,
      "logits/rejected": -0.5513606667518616,
      "logps/chosen": -5.59161376953125,
      "logps/rejected": -47.29106903076172,
      "loss": 0.8299980759620667,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5494524836540222,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.13214518129825592,
      "rewards/margins": 2.937675952911377,
      "rewards/rejected": -3.069821357727051,
      "step": 57,
      "train_speed(iter/s)": 0.011217
    },
    {
      "epoch": 0.14586607984910405,
      "grad_norm": 4.7582502365112305,
      "learning_rate": 0.00019333333333333333,
      "logits/chosen": -0.4154261350631714,
      "logits/rejected": -0.5971646904945374,
      "logps/chosen": -3.059924364089966,
      "logps/rejected": -43.29597091674805,
      "loss": 0.5003064274787903,
      "memory(GiB)": 40.85,
      "nll_loss": 0.25123852491378784,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07654222846031189,
      "rewards/margins": 2.7813808917999268,
      "rewards/rejected": -2.704838752746582,
      "step": 58,
      "train_speed(iter/s)": 0.011219
    },
    {
      "epoch": 0.14838101226029551,
      "grad_norm": 2.1735236644744873,
      "learning_rate": 0.00019666666666666666,
      "logits/chosen": -0.4227810800075531,
      "logits/rejected": -0.47660815715789795,
      "logps/chosen": -8.593082427978516,
      "logps/rejected": -26.549867630004883,
      "loss": 1.0610322952270508,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5426512360572815,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.10025864094495773,
      "rewards/margins": 1.2511481046676636,
      "rewards/rejected": -1.3514068126678467,
      "step": 59,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.15089594467148695,
      "grad_norm": 2.805598258972168,
      "learning_rate": 0.0002,
      "logits/chosen": -0.300743043422699,
      "logits/rejected": -0.47928643226623535,
      "logps/chosen": -6.593828201293945,
      "logps/rejected": -33.23126983642578,
      "loss": 0.7988956570625305,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40749770402908325,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.0006714202463626862,
      "rewards/margins": 1.6283259391784668,
      "rewards/rejected": -1.6276545524597168,
      "step": 60,
      "train_speed(iter/s)": 0.011221
    },
    {
      "epoch": 0.1534108770826784,
      "grad_norm": 2.290149211883545,
      "learning_rate": 0.0001999996142159566,
      "logits/chosen": -0.3385659158229828,
      "logits/rejected": -0.4102441966533661,
      "logps/chosen": -8.33851146697998,
      "logps/rejected": -29.508546829223633,
      "loss": 0.7559148073196411,
      "memory(GiB)": 40.85,
      "nll_loss": 0.35449326038360596,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0228387713432312,
      "rewards/margins": 1.5235671997070312,
      "rewards/rejected": -1.5464060306549072,
      "step": 61,
      "train_speed(iter/s)": 0.011222
    },
    {
      "epoch": 0.15592580949386986,
      "grad_norm": 1.7177425622940063,
      "learning_rate": 0.0001999984568668029,
      "logits/chosen": -0.32937324047088623,
      "logits/rejected": -0.4484848380088806,
      "logps/chosen": -3.493271827697754,
      "logps/rejected": -34.78036880493164,
      "loss": 0.6279686093330383,
      "memory(GiB)": 40.85,
      "nll_loss": 0.2725304365158081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11392105370759964,
      "rewards/margins": 1.8176403045654297,
      "rewards/rejected": -1.703719139099121,
      "step": 62,
      "train_speed(iter/s)": 0.011223
    },
    {
      "epoch": 0.1584407419050613,
      "grad_norm": 1.8914873600006104,
      "learning_rate": 0.00019999652796146876,
      "logits/chosen": -0.29077184200286865,
      "logits/rejected": -0.48235204815864563,
      "logps/chosen": -7.319096565246582,
      "logps/rejected": -39.54444885253906,
      "loss": 0.7298395037651062,
      "memory(GiB)": 40.85,
      "nll_loss": 0.40350908041000366,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0820951834321022,
      "rewards/margins": 1.882514476776123,
      "rewards/rejected": -1.8004192113876343,
      "step": 63,
      "train_speed(iter/s)": 0.011224
    },
    {
      "epoch": 0.16095567431625274,
      "grad_norm": 1.429303526878357,
      "learning_rate": 0.0001999938275148369,
      "logits/chosen": -0.3300917446613312,
      "logits/rejected": -0.45771050453186035,
      "logps/chosen": -3.8289737701416016,
      "logps/rejected": -39.278968811035156,
      "loss": 0.39868107438087463,
      "memory(GiB)": 40.85,
      "nll_loss": 0.18221822381019592,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.07021164149045944,
      "rewards/margins": 2.249926805496216,
      "rewards/rejected": -2.320138454437256,
      "step": 64,
      "train_speed(iter/s)": 0.011226
    },
    {
      "epoch": 0.1634706067274442,
      "grad_norm": 2.639005661010742,
      "learning_rate": 0.00019999035554774314,
      "logits/chosen": -0.36244961619377136,
      "logits/rejected": -0.5052404403686523,
      "logps/chosen": -5.230596542358398,
      "logps/rejected": -39.43718338012695,
      "loss": 0.7721049189567566,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4854365885257721,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.1088234931230545,
      "rewards/margins": 2.1535189151763916,
      "rewards/rejected": -2.2623424530029297,
      "step": 65,
      "train_speed(iter/s)": 0.011228
    },
    {
      "epoch": 0.16598553913863565,
      "grad_norm": 2.576199769973755,
      "learning_rate": 0.00019998611208697607,
      "logits/chosen": -0.3720204830169678,
      "logits/rejected": -0.507696270942688,
      "logps/chosen": -4.436941146850586,
      "logps/rejected": -34.991371154785156,
      "loss": 0.7472538352012634,
      "memory(GiB)": 40.85,
      "nll_loss": 0.39483439922332764,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.0759558230638504,
      "rewards/margins": 1.7400535345077515,
      "rewards/rejected": -1.816009521484375,
      "step": 66,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.16850047154982709,
      "grad_norm": 1.724285364151001,
      "learning_rate": 0.00019998109716527686,
      "logits/chosen": -0.25216248631477356,
      "logits/rejected": -0.4494931101799011,
      "logps/chosen": -3.201148748397827,
      "logps/rejected": -39.95793533325195,
      "loss": 0.4833192229270935,
      "memory(GiB)": 40.85,
      "nll_loss": 0.20408841967582703,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06092430651187897,
      "rewards/margins": 2.2583165168762207,
      "rewards/rejected": -2.197392225265503,
      "step": 67,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.17101540396101855,
      "grad_norm": 2.91644287109375,
      "learning_rate": 0.00019997531082133904,
      "logits/chosen": -0.30811840295791626,
      "logits/rejected": -0.413838654756546,
      "logps/chosen": -7.5586838722229,
      "logps/rejected": -29.464954376220703,
      "loss": 1.0136064291000366,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5009399652481079,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0595913827419281,
      "rewards/margins": 1.2450876235961914,
      "rewards/rejected": -1.304679036140442,
      "step": 68,
      "train_speed(iter/s)": 0.011229
    },
    {
      "epoch": 0.17353033637221,
      "grad_norm": 1.6453865766525269,
      "learning_rate": 0.00019996875309980826,
      "logits/chosen": -0.32626980543136597,
      "logits/rejected": -0.47710081934928894,
      "logps/chosen": -4.140695095062256,
      "logps/rejected": -33.830894470214844,
      "loss": 0.6451548933982849,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3717566728591919,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.094700388610363,
      "rewards/margins": 1.8348017930984497,
      "rewards/rejected": -1.7401013374328613,
      "step": 69,
      "train_speed(iter/s)": 0.01123
    },
    {
      "epoch": 0.17604526878340146,
      "grad_norm": 2.180673360824585,
      "learning_rate": 0.0001999614240512817,
      "logits/chosen": -0.2670721411705017,
      "logits/rejected": -0.46321332454681396,
      "logps/chosen": -7.102023601531982,
      "logps/rejected": -27.861366271972656,
      "loss": 0.7481832504272461,
      "memory(GiB)": 40.85,
      "nll_loss": 0.37576019763946533,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16741110384464264,
      "rewards/margins": 1.3596775531768799,
      "rewards/rejected": -1.1922664642333984,
      "step": 70,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.1785602011945929,
      "grad_norm": 2.049133539199829,
      "learning_rate": 0.00019995332373230808,
      "logits/chosen": -0.241715669631958,
      "logits/rejected": -0.407115638256073,
      "logps/chosen": -4.861550331115723,
      "logps/rejected": -34.9086799621582,
      "loss": 0.6162200570106506,
      "memory(GiB)": 40.85,
      "nll_loss": 0.25812438130378723,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.037743180990219116,
      "rewards/margins": 1.569511890411377,
      "rewards/rejected": -1.5317686796188354,
      "step": 71,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.18107513360578434,
      "grad_norm": 4.052990913391113,
      "learning_rate": 0.00019994445220538677,
      "logits/chosen": -0.2541053295135498,
      "logits/rejected": -0.36276954412460327,
      "logps/chosen": -7.95726203918457,
      "logps/rejected": -36.47919845581055,
      "loss": 0.9209573268890381,
      "memory(GiB)": 40.85,
      "nll_loss": 0.5536596179008484,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.17974892258644104,
      "rewards/margins": 1.929343581199646,
      "rewards/rejected": -2.1090924739837646,
      "step": 72,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.1835900660169758,
      "grad_norm": 3.210548162460327,
      "learning_rate": 0.0001999348095389677,
      "logits/chosen": -0.24167482554912567,
      "logits/rejected": -0.45636847615242004,
      "logps/chosen": -7.025866508483887,
      "logps/rejected": -48.629905700683594,
      "loss": 0.7358823418617249,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4621264636516571,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.05025668814778328,
      "rewards/margins": 2.2894766330718994,
      "rewards/rejected": -2.339733123779297,
      "step": 73,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.18610499842816725,
      "grad_norm": 4.565255165100098,
      "learning_rate": 0.0001999243958074506,
      "logits/chosen": -0.21860051155090332,
      "logits/rejected": -0.42088112235069275,
      "logps/chosen": -13.931788444519043,
      "logps/rejected": -42.80027770996094,
      "loss": 1.2019851207733154,
      "memory(GiB)": 40.85,
      "nll_loss": 0.7477802038192749,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.36434897780418396,
      "rewards/margins": 1.4156924486160278,
      "rewards/rejected": -1.7800413370132446,
      "step": 74,
      "train_speed(iter/s)": 0.011231
    },
    {
      "epoch": 0.18861993083935868,
      "grad_norm": 2.291332244873047,
      "learning_rate": 0.00019991321109118448,
      "logits/chosen": -0.39398425817489624,
      "logits/rejected": -0.44725537300109863,
      "logps/chosen": -6.393754482269287,
      "logps/rejected": -27.47494888305664,
      "loss": 0.7488479018211365,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3586839437484741,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.012238548137247562,
      "rewards/margins": 1.2588202953338623,
      "rewards/rejected": -1.246581792831421,
      "step": 75,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.19113486325055015,
      "grad_norm": 1.830254077911377,
      "learning_rate": 0.00019990125547646704,
      "logits/chosen": -0.31489449739456177,
      "logits/rejected": -0.47091639041900635,
      "logps/chosen": -2.980022668838501,
      "logps/rejected": -29.753929138183594,
      "loss": 0.6747189164161682,
      "memory(GiB)": 40.85,
      "nll_loss": 0.2754742503166199,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03740588575601578,
      "rewards/margins": 1.4201977252960205,
      "rewards/rejected": -1.3827917575836182,
      "step": 76,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.1936497956617416,
      "grad_norm": 1.903449296951294,
      "learning_rate": 0.00019988852905554404,
      "logits/chosen": -0.2765476703643799,
      "logits/rejected": -0.4408729076385498,
      "logps/chosen": -5.630273342132568,
      "logps/rejected": -31.533220291137695,
      "loss": 0.7030317783355713,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3307255208492279,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1330985128879547,
      "rewards/margins": 1.2641156911849976,
      "rewards/rejected": -1.1310172080993652,
      "step": 77,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.19616472807293303,
      "grad_norm": 1.736433744430542,
      "learning_rate": 0.0001998750319266084,
      "logits/chosen": -0.3374086618423462,
      "logits/rejected": -0.48207730054855347,
      "logps/chosen": -5.3590593338012695,
      "logps/rejected": -29.23048973083496,
      "loss": 0.8495813608169556,
      "memory(GiB)": 40.85,
      "nll_loss": 0.37136876583099365,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.09036409854888916,
      "rewards/margins": 1.2102724313735962,
      "rewards/rejected": -1.119908332824707,
      "step": 78,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.1986796604841245,
      "grad_norm": 2.0839672088623047,
      "learning_rate": 0.00019986076419379974,
      "logits/chosen": -0.26226145029067993,
      "logits/rejected": -0.4474099576473236,
      "logps/chosen": -7.3566060066223145,
      "logps/rejected": -40.514286041259766,
      "loss": 0.9187750220298767,
      "memory(GiB)": 40.85,
      "nll_loss": 0.60602867603302,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09458937495946884,
      "rewards/margins": 1.9139156341552734,
      "rewards/rejected": -1.8193262815475464,
      "step": 79,
      "train_speed(iter/s)": 0.011232
    },
    {
      "epoch": 0.20119459289531594,
      "grad_norm": 2.656200885772705,
      "learning_rate": 0.00019984572596720324,
      "logits/chosen": -0.4218776226043701,
      "logits/rejected": -0.548494279384613,
      "logps/chosen": -5.5665717124938965,
      "logps/rejected": -34.339046478271484,
      "loss": 0.7904801964759827,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4805757403373718,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.07708686590194702,
      "rewards/margins": 1.846845269203186,
      "rewards/rejected": -1.9239320755004883,
      "step": 80,
      "train_speed(iter/s)": 0.011233
    },
    {
      "epoch": 0.20370952530650738,
      "grad_norm": 1.582141399383545,
      "learning_rate": 0.00019982991736284915,
      "logits/chosen": -0.384469211101532,
      "logits/rejected": -0.48424291610717773,
      "logps/chosen": -6.730848789215088,
      "logps/rejected": -35.332828521728516,
      "loss": 0.6736842393875122,
      "memory(GiB)": 40.85,
      "nll_loss": 0.38718143105506897,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.02753191813826561,
      "rewards/margins": 2.01827073097229,
      "rewards/rejected": -1.9907389879226685,
      "step": 81,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.20622445771769884,
      "grad_norm": 10.319942474365234,
      "learning_rate": 0.00019981333850271158,
      "logits/chosen": -0.3957505226135254,
      "logits/rejected": -0.5100387334823608,
      "logps/chosen": -7.3371148109436035,
      "logps/rejected": -40.714134216308594,
      "loss": 0.7473828196525574,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4370650351047516,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.05588718503713608,
      "rewards/margins": 2.654069423675537,
      "rewards/rejected": -2.709956645965576,
      "step": 82,
      "train_speed(iter/s)": 0.011234
    },
    {
      "epoch": 0.20873939012889028,
      "grad_norm": 2.3877530097961426,
      "learning_rate": 0.0001997959895147077,
      "logits/chosen": -0.36577096581459045,
      "logits/rejected": -0.5359358787536621,
      "logps/chosen": -4.45527458190918,
      "logps/rejected": -46.861942291259766,
      "loss": 0.5462505221366882,
      "memory(GiB)": 40.85,
      "nll_loss": 0.30273228883743286,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15123440325260162,
      "rewards/margins": 2.9262986183166504,
      "rewards/rejected": -2.77506422996521,
      "step": 83,
      "train_speed(iter/s)": 0.011235
    },
    {
      "epoch": 0.21125432254008172,
      "grad_norm": 2.1806390285491943,
      "learning_rate": 0.0001997778705326968,
      "logits/chosen": -0.3485352396965027,
      "logits/rejected": -0.5878627300262451,
      "logps/chosen": -2.89451265335083,
      "logps/rejected": -44.27880859375,
      "loss": 0.6081856489181519,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3298896849155426,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05770353972911835,
      "rewards/margins": 2.5825228691101074,
      "rewards/rejected": -2.5248193740844727,
      "step": 84,
      "train_speed(iter/s)": 0.011235
    },
    {
      "epoch": 0.2137692549512732,
      "grad_norm": 2.80009388923645,
      "learning_rate": 0.00019975898169647915,
      "logits/chosen": -0.39257609844207764,
      "logits/rejected": -0.5535434484481812,
      "logps/chosen": -7.839548587799072,
      "logps/rejected": -42.132286071777344,
      "loss": 0.7175056338310242,
      "memory(GiB)": 40.85,
      "nll_loss": 0.405160129070282,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.025028400123119354,
      "rewards/margins": 2.475644111633301,
      "rewards/rejected": -2.450615644454956,
      "step": 85,
      "train_speed(iter/s)": 0.011235
    },
    {
      "epoch": 0.21628418736246463,
      "grad_norm": 3.3921735286712646,
      "learning_rate": 0.000199739323151795,
      "logits/chosen": -0.35620519518852234,
      "logits/rejected": -0.6006304025650024,
      "logps/chosen": -5.945646286010742,
      "logps/rejected": -45.728271484375,
      "loss": 0.6699403524398804,
      "memory(GiB)": 40.85,
      "nll_loss": 0.33112284541130066,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.012618173845112324,
      "rewards/margins": 2.700666666030884,
      "rewards/rejected": -2.713285207748413,
      "step": 86,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.21879911977365607,
      "grad_norm": 3.2411773204803467,
      "learning_rate": 0.00019971889505032334,
      "logits/chosen": -0.42666560411453247,
      "logits/rejected": -0.6167599558830261,
      "logps/chosen": -5.318458080291748,
      "logps/rejected": -49.244285583496094,
      "loss": 0.6356828808784485,
      "memory(GiB)": 40.85,
      "nll_loss": 0.39307594299316406,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1333954930305481,
      "rewards/margins": 3.1458327770233154,
      "rewards/rejected": -3.2792282104492188,
      "step": 87,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.22131405218484754,
      "grad_norm": 1.924742341041565,
      "learning_rate": 0.00019969769754968098,
      "logits/chosen": -0.3413890600204468,
      "logits/rejected": -0.5464233160018921,
      "logps/chosen": -5.729033946990967,
      "logps/rejected": -41.974422454833984,
      "loss": 0.5986770987510681,
      "memory(GiB)": 40.85,
      "nll_loss": 0.3535672724246979,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14028939604759216,
      "rewards/margins": 2.51139235496521,
      "rewards/rejected": -2.371102809906006,
      "step": 88,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.22382898459603898,
      "grad_norm": 2.161083459854126,
      "learning_rate": 0.00019967573081342103,
      "logits/chosen": -0.38623473048210144,
      "logits/rejected": -0.5679455399513245,
      "logps/chosen": -6.484611511230469,
      "logps/rejected": -38.930511474609375,
      "loss": 0.7250155806541443,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4377710521221161,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.024773895740509033,
      "rewards/margins": 2.3375205993652344,
      "rewards/rejected": -2.3622944355010986,
      "step": 89,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.22634391700723044,
      "grad_norm": 6.164645195007324,
      "learning_rate": 0.00019965299501103177,
      "logits/chosen": -0.33336132764816284,
      "logits/rejected": -0.5168649554252625,
      "logps/chosen": -11.845653533935547,
      "logps/rejected": -55.690311431884766,
      "loss": 1.0204803943634033,
      "memory(GiB)": 40.85,
      "nll_loss": 0.7011927962303162,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.22887784242630005,
      "rewards/margins": 3.111534357070923,
      "rewards/rejected": -3.3404123783111572,
      "step": 90,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.22885884941842188,
      "grad_norm": 2.4981861114501953,
      "learning_rate": 0.00019962949031793542,
      "logits/chosen": -0.4224125146865845,
      "logits/rejected": -0.5427595376968384,
      "logps/chosen": -5.719327926635742,
      "logps/rejected": -33.303741455078125,
      "loss": 0.7469587326049805,
      "memory(GiB)": 40.85,
      "nll_loss": 0.4206176996231079,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.008410206064581871,
      "rewards/margins": 1.994816541671753,
      "rewards/rejected": -2.0032269954681396,
      "step": 91,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.23137378182961332,
      "grad_norm": 2.5139830112457275,
      "learning_rate": 0.00019960521691548674,
      "logits/chosen": -0.34622836112976074,
      "logits/rejected": -0.5427547097206116,
      "logps/chosen": -5.069763660430908,
      "logps/rejected": -41.81755828857422,
      "loss": 0.6454588770866394,
      "memory(GiB)": 40.85,
      "nll_loss": 0.35349082946777344,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.046083107590675354,
      "rewards/margins": 2.2620620727539062,
      "rewards/rejected": -2.30814528465271,
      "step": 92,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.2338887142408048,
      "grad_norm": 2.1890244483947754,
      "learning_rate": 0.0001995801749909715,
      "logits/chosen": -0.3081744611263275,
      "logits/rejected": -0.47990015149116516,
      "logps/chosen": -6.5690107345581055,
      "logps/rejected": -34.549991607666016,
      "loss": 0.6765730381011963,
      "memory(GiB)": 42.61,
      "nll_loss": 0.3245474696159363,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.060123831033706665,
      "rewards/margins": 1.8496551513671875,
      "rewards/rejected": -1.7895313501358032,
      "step": 93,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.23640364665199623,
      "grad_norm": 1.8172200918197632,
      "learning_rate": 0.00019955436473760525,
      "logits/chosen": -0.24766620993614197,
      "logits/rejected": -0.49844813346862793,
      "logps/chosen": -5.177490711212158,
      "logps/rejected": -48.42265701293945,
      "loss": 0.7104548811912537,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4418777823448181,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10037174820899963,
      "rewards/margins": 2.629190683364868,
      "rewards/rejected": -2.5288188457489014,
      "step": 94,
      "train_speed(iter/s)": 0.011236
    },
    {
      "epoch": 0.23891857906318767,
      "grad_norm": 2.0303428173065186,
      "learning_rate": 0.0001995277863545316,
      "logits/chosen": -0.3552500307559967,
      "logits/rejected": -0.5088093876838684,
      "logps/chosen": -3.972975730895996,
      "logps/rejected": -29.582290649414062,
      "loss": 0.5607516169548035,
      "memory(GiB)": 42.61,
      "nll_loss": 0.27718478441238403,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09949840605258942,
      "rewards/margins": 1.6923424005508423,
      "rewards/rejected": -1.592844009399414,
      "step": 95,
      "train_speed(iter/s)": 0.011237
    },
    {
      "epoch": 0.24143351147437914,
      "grad_norm": 2.3172338008880615,
      "learning_rate": 0.00019950044004682092,
      "logits/chosen": -0.30597299337387085,
      "logits/rejected": -0.4608762860298157,
      "logps/chosen": -10.467157363891602,
      "logps/rejected": -44.41074752807617,
      "loss": 0.9089665412902832,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5540332794189453,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.0023995572701096535,
      "rewards/margins": 2.6161789894104004,
      "rewards/rejected": -2.6185781955718994,
      "step": 96,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.24394844388557058,
      "grad_norm": 1.6338391304016113,
      "learning_rate": 0.0001994723260254686,
      "logits/chosen": -0.20743098855018616,
      "logits/rejected": -0.4207964837551117,
      "logps/chosen": -4.458022117614746,
      "logps/rejected": -48.164268493652344,
      "loss": 0.5020785927772522,
      "memory(GiB)": 42.61,
      "nll_loss": 0.2824496626853943,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15383610129356384,
      "rewards/margins": 3.0794906616210938,
      "rewards/rejected": -2.925654172897339,
      "step": 97,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.24646337629676202,
      "grad_norm": 2.5537126064300537,
      "learning_rate": 0.0001994434445073934,
      "logits/chosen": -0.1935003101825714,
      "logits/rejected": -0.4715961217880249,
      "logps/chosen": -3.131989002227783,
      "logps/rejected": -61.07079315185547,
      "loss": 0.3960198163986206,
      "memory(GiB)": 42.61,
      "nll_loss": 0.21717773377895355,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07503610849380493,
      "rewards/margins": 4.007692337036133,
      "rewards/rejected": -3.9326562881469727,
      "step": 98,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.24897830870795348,
      "grad_norm": 7.057893753051758,
      "learning_rate": 0.00019941379571543596,
      "logits/chosen": -0.21255502104759216,
      "logits/rejected": -0.4117867946624756,
      "logps/chosen": -5.764989852905273,
      "logps/rejected": -59.11027908325195,
      "loss": 0.7163435816764832,
      "memory(GiB)": 42.61,
      "nll_loss": 0.4923526346683502,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1150842159986496,
      "rewards/margins": 3.945159912109375,
      "rewards/rejected": -4.060243606567383,
      "step": 99,
      "train_speed(iter/s)": 0.011238
    },
    {
      "epoch": 0.25149324111914495,
      "grad_norm": 6.774058818817139,
      "learning_rate": 0.00019938337987835688,
      "logits/chosen": -0.2647398114204407,
      "logits/rejected": -0.34906458854675293,
      "logps/chosen": -10.037410736083984,
      "logps/rejected": -37.84819030761719,
      "loss": 1.0176275968551636,
      "memory(GiB)": 42.61,
      "nll_loss": 0.5385105609893799,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.28268206119537354,
      "rewards/margins": 2.0650789737701416,
      "rewards/rejected": -2.3477611541748047,
      "step": 100,
      "train_speed(iter/s)": 0.011238
    }
  ],
  "logging_steps": 1,
  "max_steps": 1191,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 5.504958669520896e+16,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}