{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.819252432155658,
  "eval_steps": 300,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.00819252432155658,
      "grad_norm": 6.9504547119140625,
      "learning_rate": 1.5384615384615387e-05,
      "logits/chosen": -0.368751585483551,
      "logits/rejected": -0.5192137956619263,
      "logps/chosen": -6.009060859680176,
      "logps/rejected": -15.29100513458252,
      "loss": 1.2444509267807007,
      "memory(GiB)": 45.94,
      "nll_loss": 0.5513038039207458,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.005419
    },
    {
      "epoch": 0.01638504864311316,
      "grad_norm": 4.338643550872803,
      "learning_rate": 3.0769230769230774e-05,
      "logits/chosen": -0.28789153695106506,
      "logits/rejected": -0.44838714599609375,
      "logps/chosen": -7.449802398681641,
      "logps/rejected": -18.95979881286621,
      "loss": 1.2916792631149292,
      "memory(GiB)": 45.94,
      "nll_loss": 0.59853196144104,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.02457757296466974,
      "grad_norm": 5.146463394165039,
      "learning_rate": 4.615384615384616e-05,
      "logits/chosen": -0.37533068656921387,
      "logits/rejected": -0.5269488096237183,
      "logps/chosen": -4.474628925323486,
      "logps/rejected": -17.257675170898438,
      "loss": 1.0800914764404297,
      "memory(GiB)": 45.94,
      "nll_loss": 0.3937700092792511,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.005001109093427658,
      "rewards/margins": 0.01398632675409317,
      "rewards/rejected": -0.008985218591988087,
      "step": 3,
      "train_speed(iter/s)": 0.005544
    },
    {
      "epoch": 0.03277009728622632,
      "grad_norm": 3.9697017669677734,
      "learning_rate": 6.153846153846155e-05,
      "logits/chosen": -0.3554455041885376,
      "logits/rejected": -0.4836651384830475,
      "logps/chosen": -7.087220191955566,
      "logps/rejected": -16.969388961791992,
      "loss": 1.2551683187484741,
      "memory(GiB)": 45.94,
      "nll_loss": 0.5823322534561157,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.01054135151207447,
      "rewards/margins": 0.04771733283996582,
      "rewards/rejected": -0.05825868248939514,
      "step": 4,
      "train_speed(iter/s)": 0.005563
    },
    {
      "epoch": 0.0409626216077829,
      "grad_norm": 7.601996421813965,
      "learning_rate": 7.692307692307693e-05,
      "logits/chosen": -0.3477073013782501,
      "logits/rejected": -0.46865567564964294,
      "logps/chosen": -8.432049751281738,
      "logps/rejected": -20.41954231262207,
      "loss": 1.2856180667877197,
      "memory(GiB)": 45.94,
      "nll_loss": 0.6360925436019897,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.10297518223524094,
      "rewards/margins": 0.15188881754875183,
      "rewards/rejected": -0.25486400723457336,
      "step": 5,
      "train_speed(iter/s)": 0.005571
    },
    {
      "epoch": 0.04915514592933948,
      "grad_norm": 6.308995246887207,
      "learning_rate": 9.230769230769232e-05,
      "logits/chosen": -0.3014591932296753,
      "logits/rejected": -0.4142557978630066,
      "logps/chosen": -8.621772766113281,
      "logps/rejected": -19.327842712402344,
      "loss": 1.2817319631576538,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6402445435523987,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.10090938210487366,
      "rewards/margins": 0.18325890600681305,
      "rewards/rejected": -0.2841682732105255,
      "step": 6,
      "train_speed(iter/s)": 0.005566
    },
    {
      "epoch": 0.05734767025089606,
      "grad_norm": 4.3832902908325195,
      "learning_rate": 0.0001076923076923077,
      "logits/chosen": -0.3800317347049713,
      "logits/rejected": -0.4671918749809265,
      "logps/chosen": -5.921639442443848,
      "logps/rejected": -15.812642097473145,
      "loss": 1.0748225450515747,
      "memory(GiB)": 46.69,
      "nll_loss": 0.42862600088119507,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": -0.07157032936811447,
      "rewards/margins": 0.1520296335220337,
      "rewards/rejected": -0.22359995543956757,
      "step": 7,
      "train_speed(iter/s)": 0.005576
    },
    {
      "epoch": 0.06554019457245264,
      "grad_norm": 2.131096601486206,
      "learning_rate": 0.0001230769230769231,
      "logits/chosen": -0.326383113861084,
      "logits/rejected": -0.4235616624355316,
      "logps/chosen": -7.039735317230225,
      "logps/rejected": -15.89234733581543,
      "loss": 1.1329768896102905,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4950546622276306,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": 0.02736004814505577,
      "rewards/margins": 0.13636186718940735,
      "rewards/rejected": -0.10900183022022247,
      "step": 8,
      "train_speed(iter/s)": 0.00558
    },
    {
      "epoch": 0.07373271889400922,
      "grad_norm": 3.154919147491455,
      "learning_rate": 0.00013846153846153847,
      "logits/chosen": -0.3660247027873993,
      "logits/rejected": -0.4395582675933838,
      "logps/chosen": -7.082914352416992,
      "logps/rejected": -12.765766143798828,
      "loss": 1.1050751209259033,
      "memory(GiB)": 46.69,
      "nll_loss": 0.44545602798461914,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.05931762978434563,
      "rewards/margins": 0.0759875625371933,
      "rewards/rejected": -0.016669925302267075,
      "step": 9,
      "train_speed(iter/s)": 0.005582
    },
    {
      "epoch": 0.0819252432155658,
      "grad_norm": 2.322492837905884,
      "learning_rate": 0.00015384615384615385,
      "logits/chosen": -0.3979243040084839,
      "logits/rejected": -0.4425738453865051,
      "logps/chosen": -8.025144577026367,
      "logps/rejected": -15.784225463867188,
      "loss": 1.075645089149475,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4679878056049347,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.048429328948259354,
      "rewards/margins": 0.1987656205892563,
      "rewards/rejected": -0.15033629536628723,
      "step": 10,
      "train_speed(iter/s)": 0.005587
    },
    {
      "epoch": 0.09011776753712238,
      "grad_norm": 2.226271152496338,
      "learning_rate": 0.00016923076923076923,
      "logits/chosen": -0.2428550124168396,
      "logits/rejected": -0.4068838357925415,
      "logps/chosen": -5.73712682723999,
      "logps/rejected": -23.750782012939453,
      "loss": 0.9072864651679993,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4175054132938385,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.0929398313164711,
      "rewards/margins": 0.5025829672813416,
      "rewards/rejected": -0.40964314341545105,
      "step": 11,
      "train_speed(iter/s)": 0.005584
    },
    {
      "epoch": 0.09831029185867896,
      "grad_norm": 9.130071640014648,
      "learning_rate": 0.00018461538461538463,
      "logits/chosen": -0.3092227280139923,
      "logits/rejected": -0.4145185053348541,
      "logps/chosen": -9.414848327636719,
      "logps/rejected": -22.3564395904541,
      "loss": 1.266426920890808,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6574685573577881,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.27316582202911377,
      "rewards/margins": 0.5058421492576599,
      "rewards/rejected": -0.7790080308914185,
      "step": 12,
      "train_speed(iter/s)": 0.005586
    },
    {
      "epoch": 0.10650281618023553,
      "grad_norm": 8.017046928405762,
      "learning_rate": 0.0002,
      "logits/chosen": -0.3036244809627533,
      "logits/rejected": -0.43275412917137146,
      "logps/chosen": -8.467231750488281,
      "logps/rejected": -26.61805534362793,
      "loss": 1.2604392766952515,
      "memory(GiB)": 46.69,
      "nll_loss": 0.7518106698989868,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.2612573206424713,
      "rewards/margins": 0.6518616676330566,
      "rewards/rejected": -0.9131190180778503,
      "step": 13,
      "train_speed(iter/s)": 0.005587
    },
    {
      "epoch": 0.11469534050179211,
      "grad_norm": 2.1487388610839844,
      "learning_rate": 0.00019999075218579183,
      "logits/chosen": -0.2624781131744385,
      "logits/rejected": -0.3863448202610016,
      "logps/chosen": -8.31130313873291,
      "logps/rejected": -26.065853118896484,
      "loss": 0.9567878246307373,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4835919439792633,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.02007860317826271,
      "rewards/margins": 0.6282916069030762,
      "rewards/rejected": -0.6082130074501038,
      "step": 14,
      "train_speed(iter/s)": 0.005584
    },
    {
      "epoch": 0.1228878648233487,
      "grad_norm": 3.114581823348999,
      "learning_rate": 0.00019996301045360873,
      "logits/chosen": -0.32559511065483093,
      "logits/rejected": -0.40920543670654297,
      "logps/chosen": -7.523581027984619,
      "logps/rejected": -18.561134338378906,
      "loss": 1.0775632858276367,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5253046751022339,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.007055876310914755,
      "rewards/margins": 0.419503390789032,
      "rewards/rejected": -0.42655929923057556,
      "step": 15,
      "train_speed(iter/s)": 0.005586
    },
    {
      "epoch": 0.13108038914490527,
      "grad_norm": 3.7939462661743164,
      "learning_rate": 0.0001999167799344583,
      "logits/chosen": -0.2907140851020813,
      "logits/rejected": -0.3809354901313782,
      "logps/chosen": -6.864745616912842,
      "logps/rejected": -20.829776763916016,
      "loss": 1.0188764333724976,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5191283822059631,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.03652028366923332,
      "rewards/margins": 0.6273013353347778,
      "rewards/rejected": -0.5907810926437378,
      "step": 16,
      "train_speed(iter/s)": 0.005586
    },
    {
      "epoch": 0.13927291346646187,
      "grad_norm": 2.7766926288604736,
      "learning_rate": 0.00019985206917896563,
      "logits/chosen": -0.3052994906902313,
      "logits/rejected": -0.37751495838165283,
      "logps/chosen": -8.09772777557373,
      "logps/rejected": -18.549510955810547,
      "loss": 1.1308537721633911,
      "memory(GiB)": 46.69,
      "nll_loss": 0.564860999584198,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.022659722715616226,
      "rewards/margins": 0.4622555077075958,
      "rewards/rejected": -0.4395957589149475,
      "step": 17,
      "train_speed(iter/s)": 0.005587
    },
    {
      "epoch": 0.14746543778801843,
      "grad_norm": 3.9402501583099365,
      "learning_rate": 0.00019976889015579165,
      "logits/chosen": -0.2790537476539612,
      "logits/rejected": -0.3640768229961395,
      "logps/chosen": -10.8331298828125,
      "logps/rejected": -25.086530685424805,
      "loss": 1.1370517015457153,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6662458777427673,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.06523045897483826,
      "rewards/margins": 0.8811236619949341,
      "rewards/rejected": -0.9463540315628052,
      "step": 18,
      "train_speed(iter/s)": 0.005586
    },
    {
      "epoch": 0.15565796210957503,
      "grad_norm": 4.501293659210205,
      "learning_rate": 0.00019966725824941932,
      "logits/chosen": -0.3320932686328888,
      "logits/rejected": -0.44169852137565613,
      "logps/chosen": -10.038064956665039,
      "logps/rejected": -25.767019271850586,
      "loss": 1.2291309833526611,
      "memory(GiB)": 46.69,
      "nll_loss": 0.7401180267333984,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.24555523693561554,
      "rewards/margins": 0.8229343891143799,
      "rewards/rejected": -1.068489670753479,
      "step": 19,
      "train_speed(iter/s)": 0.005589
    },
    {
      "epoch": 0.1638504864311316,
      "grad_norm": 1.8677138090133667,
      "learning_rate": 0.00019954719225730847,
      "logits/chosen": -0.28071457147598267,
      "logits/rejected": -0.4227759838104248,
      "logps/chosen": -7.934963703155518,
      "logps/rejected": -24.492511749267578,
      "loss": 0.956333339214325,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4845323860645294,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.04675678163766861,
      "rewards/margins": 0.7908064126968384,
      "rewards/rejected": -0.8375633358955383,
      "step": 20,
      "train_speed(iter/s)": 0.005589
    },
    {
      "epoch": 0.17204301075268819,
      "grad_norm": 1.5877766609191895,
      "learning_rate": 0.00019940871438641882,
      "logits/chosen": -0.3024439215660095,
      "logits/rejected": -0.43745386600494385,
      "logps/chosen": -6.4543776512146,
      "logps/rejected": -22.530338287353516,
      "loss": 0.9309813380241394,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4551314115524292,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.04532763361930847,
      "rewards/margins": 0.6945431232452393,
      "rewards/rejected": -0.6492154598236084,
      "step": 21,
      "train_speed(iter/s)": 0.005589
    },
    {
      "epoch": 0.18023553507424475,
      "grad_norm": 1.7120730876922607,
      "learning_rate": 0.00019925185024910277,
      "logits/chosen": -0.2665388584136963,
      "logits/rejected": -0.39961740374565125,
      "logps/chosen": -7.612581729888916,
      "logps/rejected": -24.12154769897461,
      "loss": 1.017120599746704,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5679780840873718,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12790349125862122,
      "rewards/margins": 0.8537773489952087,
      "rewards/rejected": -0.7258739471435547,
      "step": 22,
      "train_speed(iter/s)": 0.005587
    },
    {
      "epoch": 0.18842805939580134,
      "grad_norm": 1.8607813119888306,
      "learning_rate": 0.00019907662885836836,
      "logits/chosen": -0.29434627294540405,
      "logits/rejected": -0.38500675559043884,
      "logps/chosen": -9.917289733886719,
      "logps/rejected": -20.862939834594727,
      "loss": 1.072438359260559,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5891656875610352,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.050024691969156265,
      "rewards/margins": 0.6231139302253723,
      "rewards/rejected": -0.5730893015861511,
      "step": 23,
      "train_speed(iter/s)": 0.005586
    },
    {
      "epoch": 0.1966205837173579,
      "grad_norm": 2.2947323322296143,
      "learning_rate": 0.00019888308262251285,
      "logits/chosen": -0.2984248995780945,
      "logits/rejected": -0.4263286292552948,
      "logps/chosen": -5.695917129516602,
      "logps/rejected": -27.542821884155273,
      "loss": 0.8626710176467896,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4990408718585968,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.05305299907922745,
      "rewards/margins": 1.2556676864624023,
      "rewards/rejected": -1.2026147842407227,
      "step": 24,
      "train_speed(iter/s)": 0.005587
    },
    {
      "epoch": 0.2048131080389145,
      "grad_norm": 2.453016996383667,
      "learning_rate": 0.0001986712473391289,
      "logits/chosen": -0.37013769149780273,
      "logits/rejected": -0.4342552423477173,
      "logps/chosen": -5.276243209838867,
      "logps/rejected": -23.596858978271484,
      "loss": 0.8703887462615967,
      "memory(GiB)": 46.69,
      "nll_loss": 0.45367568731307983,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.010003786534070969,
      "rewards/margins": 1.1414563655853271,
      "rewards/rejected": -1.1514601707458496,
      "step": 25,
      "train_speed(iter/s)": 0.005589
    },
    {
      "epoch": 0.21300563236047107,
      "grad_norm": 3.131953477859497,
      "learning_rate": 0.00019844116218848334,
      "logits/chosen": -0.3483574688434601,
      "logits/rejected": -0.4867978096008301,
      "logps/chosen": -6.131460666656494,
      "logps/rejected": -25.60129165649414,
      "loss": 0.8633096814155579,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4731563329696655,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.08134596049785614,
      "rewards/margins": 1.066148042678833,
      "rewards/rejected": -0.9848020076751709,
      "step": 26,
      "train_speed(iter/s)": 0.005591
    },
    {
      "epoch": 0.22119815668202766,
      "grad_norm": 2.4780352115631104,
      "learning_rate": 0.00019819286972627066,
      "logits/chosen": -0.36855828762054443,
      "logits/rejected": -0.49951156973838806,
      "logps/chosen": -5.627861499786377,
      "logps/rejected": -31.835105895996094,
      "loss": 0.7191118597984314,
      "memory(GiB)": 46.69,
      "nll_loss": 0.39769047498703003,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.007685612887144089,
      "rewards/margins": 1.6878087520599365,
      "rewards/rejected": -1.6954944133758545,
      "step": 27,
      "train_speed(iter/s)": 0.005593
    },
    {
      "epoch": 0.22939068100358423,
      "grad_norm": 2.9881274700164795,
      "learning_rate": 0.00019792641587574212,
      "logits/chosen": -0.2455301731824875,
      "logits/rejected": -0.39179977774620056,
      "logps/chosen": -6.589425563812256,
      "logps/rejected": -36.522743225097656,
      "loss": 0.7601451277732849,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4516022801399231,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.009008985012769699,
      "rewards/margins": 1.831702709197998,
      "rewards/rejected": -1.8226937055587769,
      "step": 28,
      "train_speed(iter/s)": 0.005593
    },
    {
      "epoch": 0.23758320532514082,
      "grad_norm": 2.97683048248291,
      "learning_rate": 0.00019764184991921178,
      "logits/chosen": -0.30706074833869934,
      "logits/rejected": -0.44865599274635315,
      "logps/chosen": -10.063987731933594,
      "logps/rejected": -31.307193756103516,
      "loss": 1.0377051830291748,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6944084763526917,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.10418727993965149,
      "rewards/margins": 1.5954655408859253,
      "rewards/rejected": -1.6996527910232544,
      "step": 29,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.2457757296466974,
      "grad_norm": 1.9776651859283447,
      "learning_rate": 0.0001973392244889415,
      "logits/chosen": -0.32163313031196594,
      "logits/rejected": -0.4449259042739868,
      "logps/chosen": -8.357927322387695,
      "logps/rejected": -33.13959884643555,
      "loss": 0.880458414554596,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5416579246520996,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04613646864891052,
      "rewards/margins": 1.6892749071121216,
      "rewards/rejected": -1.735411286354065,
      "step": 30,
      "train_speed(iter/s)": 0.005595
    },
    {
      "epoch": 0.25396825396825395,
      "grad_norm": 2.918856620788574,
      "learning_rate": 0.00019701859555740648,
      "logits/chosen": -0.3250303566455841,
      "logits/rejected": -0.42262473702430725,
      "logps/chosen": -7.8114013671875,
      "logps/rejected": -27.219764709472656,
      "loss": 0.9608601331710815,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5916098356246948,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.02971716970205307,
      "rewards/margins": 1.5878727436065674,
      "rewards/rejected": -1.5581557750701904,
      "step": 31,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.26216077828981055,
      "grad_norm": 1.8905093669891357,
      "learning_rate": 0.00019668002242694238,
      "logits/chosen": -0.2707211673259735,
      "logits/rejected": -0.4425659477710724,
      "logps/chosen": -5.811425685882568,
      "logps/rejected": -30.51934814453125,
      "loss": 0.7490611672401428,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3751787841320038,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11003397405147552,
      "rewards/margins": 1.2447575330734253,
      "rewards/rejected": -1.134723424911499,
      "step": 32,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.27035330261136714,
      "grad_norm": 3.2241005897521973,
      "learning_rate": 0.00019632356771877735,
      "logits/chosen": -0.31523844599723816,
      "logits/rejected": -0.48877376317977905,
      "logps/chosen": -7.238929271697998,
      "logps/rejected": -24.841379165649414,
      "loss": 0.9657001495361328,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5450810194015503,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.09001664817333221,
      "rewards/margins": 0.8783948421478271,
      "rewards/rejected": -0.7883782386779785,
      "step": 33,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.27854582693292373,
      "grad_norm": 2.6415717601776123,
      "learning_rate": 0.00019594929736144976,
      "logits/chosen": -0.370350182056427,
      "logits/rejected": -0.48758062720298767,
      "logps/chosen": -6.904268741607666,
      "logps/rejected": -27.709651947021484,
      "loss": 0.7636420130729675,
      "memory(GiB)": 46.69,
      "nll_loss": 0.37949565052986145,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.012274645268917084,
      "rewards/margins": 1.4130356311798096,
      "rewards/rejected": -1.4007611274719238,
      "step": 34,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.2867383512544803,
      "grad_norm": 1.870375156402588,
      "learning_rate": 0.0001955572805786141,
      "logits/chosen": -0.32100898027420044,
      "logits/rejected": -0.5096184611320496,
      "logps/chosen": -5.18679666519165,
      "logps/rejected": -32.468994140625,
      "loss": 0.7190079689025879,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4115956127643585,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.07185505330562592,
      "rewards/margins": 1.6462374925613403,
      "rewards/rejected": -1.5743825435638428,
      "step": 35,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.29493087557603687,
      "grad_norm": 6.671431064605713,
      "learning_rate": 0.00019514758987623784,
      "logits/chosen": -0.34625619649887085,
      "logits/rejected": -0.4750059247016907,
      "logps/chosen": -10.31639575958252,
      "logps/rejected": -36.582740783691406,
      "loss": 1.0590507984161377,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6509675979614258,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.20210294425487518,
      "rewards/margins": 1.9486511945724487,
      "rewards/rejected": -2.15075421333313,
      "step": 36,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.30312339989759346,
      "grad_norm": 7.133448123931885,
      "learning_rate": 0.000194720301029191,
      "logits/chosen": -0.3258736729621887,
      "logits/rejected": -0.5075902938842773,
      "logps/chosen": -9.880843162536621,
      "logps/rejected": -39.184967041015625,
      "loss": 1.0690901279449463,
      "memory(GiB)": 46.69,
      "nll_loss": 0.7646387815475464,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -0.22141867876052856,
      "rewards/margins": 2.19596004486084,
      "rewards/rejected": -2.4173786640167236,
      "step": 37,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.31131592421915005,
      "grad_norm": 3.9345178604125977,
      "learning_rate": 0.00019427549306723097,
      "logits/chosen": -0.32678377628326416,
      "logits/rejected": -0.5091384649276733,
      "logps/chosen": -6.220644950866699,
      "logps/rejected": -38.1715087890625,
      "loss": 0.7903810143470764,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5349410772323608,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04198794811964035,
      "rewards/margins": 2.360246419906616,
      "rewards/rejected": -2.31825852394104,
      "step": 38,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.3195084485407066,
      "grad_norm": 2.339552402496338,
      "learning_rate": 0.0001938132482603856,
      "logits/chosen": -0.24855449795722961,
      "logits/rejected": -0.46237537264823914,
      "logps/chosen": -4.8696136474609375,
      "logps/rejected": -35.97483825683594,
      "loss": 0.6593115925788879,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3662545680999756,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0723145455121994,
      "rewards/margins": 1.8895220756530762,
      "rewards/rejected": -1.8172073364257812,
      "step": 39,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.3277009728622632,
      "grad_norm": 2.3479816913604736,
      "learning_rate": 0.0001933336521037367,
      "logits/chosen": -0.3132964074611664,
      "logits/rejected": -0.4357830882072449,
      "logps/chosen": -7.038135528564453,
      "logps/rejected": -29.72612190246582,
      "loss": 0.7142176628112793,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4181695878505707,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.04734823852777481,
      "rewards/margins": 1.5952571630477905,
      "rewards/rejected": -1.5479090213775635,
      "step": 40,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.3358934971838198,
      "grad_norm": 2.551851272583008,
      "learning_rate": 0.00019283679330160726,
      "logits/chosen": -0.32567888498306274,
      "logits/rejected": -0.4566379487514496,
      "logps/chosen": -7.564522743225098,
      "logps/rejected": -29.297645568847656,
      "loss": 0.8506546020507812,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4627075493335724,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.018498025834560394,
      "rewards/margins": 1.5199252367019653,
      "rewards/rejected": -1.5384231805801392,
      "step": 41,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.34408602150537637,
      "grad_norm": 1.9277410507202148,
      "learning_rate": 0.00019232276375115515,
      "logits/chosen": -0.33336636424064636,
      "logits/rejected": -0.4738547205924988,
      "logps/chosen": -6.605499267578125,
      "logps/rejected": -26.275497436523438,
      "loss": 0.7640136480331421,
      "memory(GiB)": 46.69,
      "nll_loss": 0.45767274498939514,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.18768644332885742,
      "rewards/margins": 1.589571237564087,
      "rewards/rejected": -1.4018847942352295,
      "step": 42,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.3522785458269329,
      "grad_norm": 2.7057321071624756,
      "learning_rate": 0.00019179165852537596,
      "logits/chosen": -0.24734170734882355,
      "logits/rejected": -0.4288603663444519,
      "logps/chosen": -8.57066822052002,
      "logps/rejected": -35.444984436035156,
      "loss": 0.9440370798110962,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6365867853164673,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.04400603100657463,
      "rewards/margins": 1.8436404466629028,
      "rewards/rejected": -1.8876465559005737,
      "step": 43,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.3604710701484895,
      "grad_norm": 2.4904823303222656,
      "learning_rate": 0.00019124357585551872,
      "logits/chosen": -0.296947717666626,
      "logits/rejected": -0.43239718675613403,
      "logps/chosen": -8.56611442565918,
      "logps/rejected": -36.052391052246094,
      "loss": 0.8505915403366089,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5557512044906616,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0323612280189991,
      "rewards/margins": 2.1337056159973145,
      "rewards/rejected": -2.1013448238372803,
      "step": 44,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.3686635944700461,
      "grad_norm": 2.8696656227111816,
      "learning_rate": 0.00019067861711291744,
      "logits/chosen": -0.255009263753891,
      "logits/rejected": -0.4662845730781555,
      "logps/chosen": -4.24545431137085,
      "logps/rejected": -36.80644607543945,
      "loss": 0.642361044883728,
      "memory(GiB)": 46.69,
      "nll_loss": 0.373349666595459,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.03832457214593887,
      "rewards/margins": 2.0462074279785156,
      "rewards/rejected": -2.007882833480835,
      "step": 45,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.3768561187916027,
      "grad_norm": 2.3527863025665283,
      "learning_rate": 0.0001900968867902419,
      "logits/chosen": -0.29511338472366333,
      "logits/rejected": -0.4300462305545807,
      "logps/chosen": -6.108550071716309,
      "logps/rejected": -30.907669067382812,
      "loss": 0.8321690559387207,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4702669382095337,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.013932769186794758,
      "rewards/margins": 1.64110267162323,
      "rewards/rejected": -1.6550354957580566,
      "step": 46,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.38504864311315923,
      "grad_norm": 1.8884741067886353,
      "learning_rate": 0.000189498492482171,
      "logits/chosen": -0.28806138038635254,
      "logits/rejected": -0.43808045983314514,
      "logps/chosen": -5.240577697753906,
      "logps/rejected": -29.244850158691406,
      "loss": 0.6651690602302551,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3822129964828491,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.10050319880247116,
      "rewards/margins": 1.6757993698120117,
      "rewards/rejected": -1.5752960443496704,
      "step": 47,
      "train_speed(iter/s)": 0.005601
    },
    {
      "epoch": 0.3932411674347158,
      "grad_norm": 1.7884507179260254,
      "learning_rate": 0.00018888354486549237,
      "logits/chosen": -0.29420942068099976,
      "logits/rejected": -0.43043237924575806,
      "logps/chosen": -6.428597450256348,
      "logps/rejected": -32.303741455078125,
      "loss": 0.700837254524231,
      "memory(GiB)": 46.69,
      "nll_loss": 0.45000576972961426,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.12149131298065186,
      "rewards/margins": 1.8844413757324219,
      "rewards/rejected": -1.7629501819610596,
      "step": 48,
      "train_speed(iter/s)": 0.005601
    },
    {
      "epoch": 0.4014336917562724,
      "grad_norm": 4.318137168884277,
      "learning_rate": 0.00018825215767863214,
      "logits/chosen": -0.18930676579475403,
      "logits/rejected": -0.40716230869293213,
      "logps/chosen": -5.259353160858154,
      "logps/rejected": -37.35228729248047,
      "loss": 0.7124305963516235,
      "memory(GiB)": 46.69,
      "nll_loss": 0.43444421887397766,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.09258248656988144,
      "rewards/margins": 2.3026676177978516,
      "rewards/rejected": -2.210084915161133,
      "step": 49,
      "train_speed(iter/s)": 0.005601
    },
    {
      "epoch": 0.409626216077829,
      "grad_norm": 4.764008522033691,
      "learning_rate": 0.0001876044477006183,
      "logits/chosen": -0.14179718494415283,
      "logits/rejected": -0.313814252614975,
      "logps/chosen": -8.565025329589844,
      "logps/rejected": -39.398197174072266,
      "loss": 1.0041635036468506,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6235343217849731,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.16454477608203888,
      "rewards/margins": 2.0170154571533203,
      "rewards/rejected": -2.1815602779388428,
      "step": 50,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.41781874039938555,
      "grad_norm": 3.3876824378967285,
      "learning_rate": 0.00018694053472948156,
      "logits/chosen": -0.20334556698799133,
      "logits/rejected": -0.4205096662044525,
      "logps/chosen": -6.429573059082031,
      "logps/rejected": -41.929100036621094,
      "loss": 0.6693562865257263,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4370025098323822,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04353882744908333,
      "rewards/margins": 2.5084493160247803,
      "rewards/rejected": -2.4649102687835693,
      "step": 51,
      "train_speed(iter/s)": 0.005591
    },
    {
      "epoch": 0.42601126472094214,
      "grad_norm": 4.126477241516113,
      "learning_rate": 0.00018626054156009806,
      "logits/chosen": -0.2386299967765808,
      "logits/rejected": -0.3825935423374176,
      "logps/chosen": -6.486486434936523,
      "logps/rejected": -40.26316833496094,
      "loss": 0.7214045524597168,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4816324710845947,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.056279510259628296,
      "rewards/margins": 2.54030704498291,
      "rewards/rejected": -2.484027624130249,
      "step": 52,
      "train_speed(iter/s)": 0.005591
    },
    {
      "epoch": 0.43420378904249873,
      "grad_norm": 7.559770584106445,
      "learning_rate": 0.00018556459396147775,
      "logits/chosen": -0.20944397151470184,
      "logits/rejected": -0.3997128903865814,
      "logps/chosen": -8.67984676361084,
      "logps/rejected": -49.18048095703125,
      "loss": 0.8435146808624268,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6305564045906067,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.22550427913665771,
      "rewards/margins": 3.0725927352905273,
      "rewards/rejected": -3.2980968952178955,
      "step": 53,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.4423963133640553,
      "grad_norm": 7.2271199226379395,
      "learning_rate": 0.00018485282065350235,
      "logits/chosen": -0.2310609519481659,
      "logits/rejected": -0.40637755393981934,
      "logps/chosen": -7.4795732498168945,
      "logps/rejected": -51.930747985839844,
      "loss": 0.8192666172981262,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6062192916870117,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": -0.15003077685832977,
      "rewards/margins": 3.5280890464782715,
      "rewards/rejected": -3.6781198978424072,
      "step": 54,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.45058883768561186,
      "grad_norm": 5.076483249664307,
      "learning_rate": 0.00018412535328311814,
      "logits/chosen": -0.1615740954875946,
      "logits/rejected": -0.3843723237514496,
      "logps/chosen": -8.26565170288086,
      "logps/rejected": -48.85228729248047,
      "loss": 0.832518994808197,
      "memory(GiB)": 46.69,
      "nll_loss": 0.59903883934021,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -9.993277490139008e-05,
      "rewards/margins": 2.931636095046997,
      "rewards/rejected": -2.9317359924316406,
      "step": 55,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.45878136200716846,
      "grad_norm": 2.6638269424438477,
      "learning_rate": 0.0001833823263999867,
      "logits/chosen": -0.14684350788593292,
      "logits/rejected": -0.34851011633872986,
      "logps/chosen": -6.972771167755127,
      "logps/rejected": -44.60012435913086,
      "loss": 0.8128335475921631,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5632097721099854,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.018600676208734512,
      "rewards/margins": 2.8621859550476074,
      "rewards/rejected": -2.84358549118042,
      "step": 56,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.46697388632872505,
      "grad_norm": 1.7933253049850464,
      "learning_rate": 0.0001826238774315995,
      "logits/chosen": -0.22660666704177856,
      "logits/rejected": -0.3997350335121155,
      "logps/chosen": -4.635244846343994,
      "logps/rejected": -39.17095947265625,
      "loss": 0.6400424242019653,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3968124985694885,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.16884273290634155,
      "rewards/margins": 2.504216432571411,
      "rewards/rejected": -2.335373640060425,
      "step": 57,
      "train_speed(iter/s)": 0.005593
    },
    {
      "epoch": 0.47516641065028165,
      "grad_norm": 2.3891918659210205,
      "learning_rate": 0.00018185014665785936,
      "logits/chosen": -0.11503090709447861,
      "logits/rejected": -0.3457573652267456,
      "logps/chosen": -5.839378356933594,
      "logps/rejected": -38.64885330200195,
      "loss": 0.8351156711578369,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5349250435829163,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11689441651105881,
      "rewards/margins": 2.108403205871582,
      "rewards/rejected": -1.991508960723877,
      "step": 58,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.4833589349718382,
      "grad_norm": 1.8465185165405273,
      "learning_rate": 0.00018106127718513516,
      "logits/chosen": -0.17817801237106323,
      "logits/rejected": -0.3191560208797455,
      "logps/chosen": -5.835081577301025,
      "logps/rejected": -35.51255798339844,
      "loss": 0.6248178482055664,
      "memory(GiB)": 46.69,
      "nll_loss": 0.36304396390914917,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.10749153047800064,
      "rewards/margins": 2.040304183959961,
      "rewards/rejected": -1.9328124523162842,
      "step": 59,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.4915514592933948,
      "grad_norm": 2.3336682319641113,
      "learning_rate": 0.00018025741491979326,
      "logits/chosen": -0.29990077018737793,
      "logits/rejected": -0.4385133981704712,
      "logps/chosen": -5.493274688720703,
      "logps/rejected": -39.10345458984375,
      "loss": 0.6828079223632812,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4555109143257141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1612246334552765,
      "rewards/margins": 2.3001227378845215,
      "rewards/rejected": -2.1388983726501465,
      "step": 60,
      "train_speed(iter/s)": 0.005592
    },
    {
      "epoch": 0.49974398361495137,
      "grad_norm": 1.7128206491470337,
      "learning_rate": 0.00017943870854121124,
      "logits/chosen": -0.29102638363838196,
      "logits/rejected": -0.41164785623550415,
      "logps/chosen": -5.145498752593994,
      "logps/rejected": -37.70575714111328,
      "loss": 0.6163212060928345,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3928971290588379,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.15216323733329773,
      "rewards/margins": 2.5461089611053467,
      "rewards/rejected": -2.3939456939697266,
      "step": 61,
      "train_speed(iter/s)": 0.005593
    },
    {
      "epoch": 0.5079365079365079,
      "grad_norm": 2.3307297229766846,
      "learning_rate": 0.00017860530947427875,
      "logits/chosen": -0.32119452953338623,
      "logits/rejected": -0.4902888238430023,
      "logps/chosen": -4.7407121658325195,
      "logps/rejected": -44.525733947753906,
      "loss": 0.6137937307357788,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3777918815612793,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.008498485200107098,
      "rewards/margins": 2.915428400039673,
      "rewards/rejected": -2.9069297313690186,
      "step": 62,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.5161290322580645,
      "grad_norm": 4.084574222564697,
      "learning_rate": 0.00017775737186139038,
      "logits/chosen": -0.21881335973739624,
      "logits/rejected": -0.47232887148857117,
      "logps/chosen": -6.758890151977539,
      "logps/rejected": -50.4245719909668,
      "loss": 0.669992208480835,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4888433814048767,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.03637097403407097,
      "rewards/margins": 3.5680360794067383,
      "rewards/rejected": -3.5316648483276367,
      "step": 63,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.5243215565796211,
      "grad_norm": 2.3942766189575195,
      "learning_rate": 0.0001768950525339362,
      "logits/chosen": -0.14836789667606354,
      "logits/rejected": -0.4676767587661743,
      "logps/chosen": -3.7084708213806152,
      "logps/rejected": -68.53382873535156,
      "loss": 0.4841669499874115,
      "memory(GiB)": 46.69,
      "nll_loss": 0.33256208896636963,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.040049366652965546,
      "rewards/margins": 4.963237285614014,
      "rewards/rejected": -4.923187732696533,
      "step": 64,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.5325140809011777,
      "grad_norm": 5.6252241134643555,
      "learning_rate": 0.00017601851098329483,
      "logits/chosen": -0.2383272349834442,
      "logits/rejected": -0.4961923062801361,
      "logps/chosen": -9.10733699798584,
      "logps/rejected": -69.8805160522461,
      "loss": 0.7837992906570435,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6463965177536011,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02835194021463394,
      "rewards/margins": 5.352425575256348,
      "rewards/rejected": -5.324073791503906,
      "step": 65,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.5407066052227343,
      "grad_norm": 3.746506929397583,
      "learning_rate": 0.00017512790933133437,
      "logits/chosen": -0.19862349331378937,
      "logits/rejected": -0.44604504108428955,
      "logps/chosen": -6.812467098236084,
      "logps/rejected": -62.03723907470703,
      "loss": 0.6775254011154175,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4538717567920685,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04974764212965965,
      "rewards/margins": 4.575608730316162,
      "rewards/rejected": -4.525860786437988,
      "step": 66,
      "train_speed(iter/s)": 0.005594
    },
    {
      "epoch": 0.5488991295442909,
      "grad_norm": 1.9380849599838257,
      "learning_rate": 0.000174223412300427,
      "logits/chosen": -0.2870069444179535,
      "logits/rejected": -0.49838078022003174,
      "logps/chosen": -4.964132308959961,
      "logps/rejected": -62.411376953125,
      "loss": 0.5415135622024536,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3554508686065674,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.0713961273431778,
      "rewards/margins": 4.775064468383789,
      "rewards/rejected": -4.703668117523193,
      "step": 67,
      "train_speed(iter/s)": 0.005595
    },
    {
      "epoch": 0.5570916538658475,
      "grad_norm": 2.481924295425415,
      "learning_rate": 0.00017330518718298264,
      "logits/chosen": -0.2851252257823944,
      "logits/rejected": -0.44593459367752075,
      "logps/chosen": -7.28018856048584,
      "logps/rejected": -50.08050537109375,
      "loss": 0.7013931274414062,
      "memory(GiB)": 46.69,
      "nll_loss": 0.46438127756118774,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.05084916949272156,
      "rewards/margins": 3.5459792613983154,
      "rewards/rejected": -3.495130777359009,
      "step": 68,
      "train_speed(iter/s)": 0.005595
    },
    {
      "epoch": 0.565284178187404,
      "grad_norm": 2.377530574798584,
      "learning_rate": 0.00017237340381050703,
      "logits/chosen": -0.23837527632713318,
      "logits/rejected": -0.45363929867744446,
      "logps/chosen": -7.0486907958984375,
      "logps/rejected": -53.989742279052734,
      "loss": 0.7356840372085571,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4811277985572815,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0028972141444683075,
      "rewards/margins": 3.634432315826416,
      "rewards/rejected": -3.6373291015625,
      "step": 69,
      "train_speed(iter/s)": 0.005595
    },
    {
      "epoch": 0.5734767025089605,
      "grad_norm": 4.029879570007324,
      "learning_rate": 0.00017142823452219038,
      "logits/chosen": -0.3782976567745209,
      "logits/rejected": -0.5180714726448059,
      "logps/chosen": -5.851866245269775,
      "logps/rejected": -45.21742248535156,
      "loss": 0.6551123857498169,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4019528925418854,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.03804052993655205,
      "rewards/margins": 3.155877113342285,
      "rewards/rejected": -3.1178367137908936,
      "step": 70,
      "train_speed(iter/s)": 0.005595
    },
    {
      "epoch": 0.5816692268305171,
      "grad_norm": 1.3280060291290283,
      "learning_rate": 0.00017046985413303215,
      "logits/chosen": -0.2792169451713562,
      "logits/rejected": -0.47711247205734253,
      "logps/chosen": -6.310469627380371,
      "logps/rejected": -51.31058120727539,
      "loss": 0.6356867551803589,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4532567858695984,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.12982985377311707,
      "rewards/margins": 3.86684513092041,
      "rewards/rejected": -3.7370152473449707,
      "step": 71,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.5898617511520737,
      "grad_norm": 6.06371545791626,
      "learning_rate": 0.00016949843990150796,
      "logits/chosen": -0.2690970003604889,
      "logits/rejected": -0.5049694776535034,
      "logps/chosen": -6.127170085906982,
      "logps/rejected": -55.83935546875,
      "loss": 0.7382916212081909,
      "memory(GiB)": 46.69,
      "nll_loss": 0.45698025822639465,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.07403797656297684,
      "rewards/margins": 3.589578866958618,
      "rewards/rejected": -3.66361665725708,
      "step": 72,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.5980542754736303,
      "grad_norm": 2.3506956100463867,
      "learning_rate": 0.00016851417149678444,
      "logits/chosen": -0.2961362302303314,
      "logits/rejected": -0.4737069606781006,
      "logps/chosen": -6.219717025756836,
      "logps/rejected": -44.68914031982422,
      "loss": 0.6626992225646973,
      "memory(GiB)": 46.69,
      "nll_loss": 0.45077890157699585,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.09962359815835953,
      "rewards/margins": 2.9673054218292236,
      "rewards/rejected": -2.8676817417144775,
      "step": 73,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.6062467997951869,
      "grad_norm": 1.1544229984283447,
      "learning_rate": 0.00016751723096548835,
      "logits/chosen": -0.40799474716186523,
      "logits/rejected": -0.5058730840682983,
      "logps/chosen": -5.0622429847717285,
      "logps/rejected": -40.20547103881836,
      "loss": 0.47565728425979614,
      "memory(GiB)": 46.69,
      "nll_loss": 0.31371772289276123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.11073534190654755,
      "rewards/margins": 2.8409810066223145,
      "rewards/rejected": -2.730245351791382,
      "step": 74,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.6144393241167435,
      "grad_norm": 6.4909563064575195,
      "learning_rate": 0.00016650780269803587,
      "logits/chosen": -0.23205381631851196,
      "logits/rejected": -0.4088100492954254,
      "logps/chosen": -8.021440505981445,
      "logps/rejected": -46.59309768676758,
      "loss": 0.6021320819854736,
      "memory(GiB)": 46.69,
      "nll_loss": 0.39458850026130676,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.11055044829845428,
      "rewards/margins": 3.0668678283691406,
      "rewards/rejected": -2.956317663192749,
      "step": 75,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.6226318484383001,
      "grad_norm": 2.430753231048584,
      "learning_rate": 0.00016548607339452853,
      "logits/chosen": -0.3427090048789978,
      "logits/rejected": -0.5122124552726746,
      "logps/chosen": -7.582193851470947,
      "logps/rejected": -44.092445373535156,
      "loss": 0.7723615765571594,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5251502394676208,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.03329634293913841,
      "rewards/margins": 3.0993239879608154,
      "rewards/rejected": -3.0660276412963867,
      "step": 76,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.6308243727598566,
      "grad_norm": 1.7724313735961914,
      "learning_rate": 0.00016445223203022166,
      "logits/chosen": -0.2575310468673706,
      "logits/rejected": -0.493635892868042,
      "logps/chosen": -6.6194748878479,
      "logps/rejected": -49.363059997558594,
      "loss": 0.7014020681381226,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5046313405036926,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11114748567342758,
      "rewards/margins": 3.2974491119384766,
      "rewards/rejected": -3.1863017082214355,
      "step": 77,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.6390168970814132,
      "grad_norm": 3.098682403564453,
      "learning_rate": 0.00016340646982057247,
      "logits/chosen": -0.2661978304386139,
      "logits/rejected": -0.4464647173881531,
      "logps/chosen": -5.315243721008301,
      "logps/rejected": -43.53657913208008,
      "loss": 0.6228019595146179,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4409627616405487,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.12046808749437332,
      "rewards/margins": 2.898390054702759,
      "rewards/rejected": -2.7779219150543213,
      "step": 78,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.6472094214029698,
      "grad_norm": 4.985314846038818,
      "learning_rate": 0.00016234898018587337,
      "logits/chosen": -0.32925689220428467,
      "logits/rejected": -0.47732216119766235,
      "logps/chosen": -6.385538101196289,
      "logps/rejected": -43.59192657470703,
      "loss": 0.7535056471824646,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5210545063018799,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09275269508361816,
      "rewards/margins": 3.1695356369018555,
      "rewards/rejected": -3.0767829418182373,
      "step": 79,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.6554019457245264,
      "grad_norm": 1.7759379148483276,
      "learning_rate": 0.0001612799587154777,
      "logits/chosen": -0.2674836218357086,
      "logits/rejected": -0.4375452995300293,
      "logps/chosen": -4.936103343963623,
      "logps/rejected": -48.7861213684082,
      "loss": 0.540897011756897,
      "memory(GiB)": 46.69,
      "nll_loss": 0.35059285163879395,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.11382850259542465,
      "rewards/margins": 3.4102067947387695,
      "rewards/rejected": -3.2963783740997314,
      "step": 80,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.663594470046083,
      "grad_norm": 7.628201007843018,
      "learning_rate": 0.00016019960313162434,
      "logits/chosen": -0.24807484447956085,
      "logits/rejected": -0.42589354515075684,
      "logps/chosen": -6.90081262588501,
      "logps/rejected": -42.77968978881836,
      "loss": 0.6282734274864197,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4406556785106659,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.15442682802677155,
      "rewards/margins": 2.8696365356445312,
      "rewards/rejected": -2.7152099609375,
      "step": 81,
      "train_speed(iter/s)": 0.005597
    },
    {
      "epoch": 0.6717869943676396,
      "grad_norm": 1.9605602025985718,
      "learning_rate": 0.00015910811325286768,
      "logits/chosen": -0.27870428562164307,
      "logits/rejected": -0.4770907163619995,
      "logps/chosen": -7.222564697265625,
      "logps/rejected": -39.67060852050781,
      "loss": 0.6897969245910645,
      "memory(GiB)": 46.69,
      "nll_loss": 0.463836133480072,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.21148104965686798,
      "rewards/margins": 2.5219335556030273,
      "rewards/rejected": -2.310452699661255,
      "step": 82,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.6799795186891961,
      "grad_norm": 1.522215723991394,
      "learning_rate": 0.00015800569095711982,
      "logits/chosen": -0.2916646897792816,
      "logits/rejected": -0.46336936950683594,
      "logps/chosen": -4.920736312866211,
      "logps/rejected": -41.507869720458984,
      "loss": 0.5498954057693481,
      "memory(GiB)": 46.69,
      "nll_loss": 0.33032387495040894,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.11536765843629837,
      "rewards/margins": 2.728242874145508,
      "rewards/rejected": -2.612875461578369,
      "step": 83,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.6881720430107527,
      "grad_norm": 1.782954454421997,
      "learning_rate": 0.00015689254014431225,
      "logits/chosen": -0.2412262111902237,
      "logits/rejected": -0.4835028052330017,
      "logps/chosen": -4.433740615844727,
      "logps/rejected": -45.75048828125,
      "loss": 0.5076733827590942,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3361254334449768,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.12744426727294922,
      "rewards/margins": 2.9612059593200684,
      "rewards/rejected": -2.83376145362854,
      "step": 84,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.6963645673323092,
      "grad_norm": 1.6077523231506348,
      "learning_rate": 0.00015576886669868296,
      "logits/chosen": -0.24575835466384888,
      "logits/rejected": -0.499642550945282,
      "logps/chosen": -6.2326812744140625,
      "logps/rejected": -50.58203887939453,
      "loss": 0.5996717810630798,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4303426146507263,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1870914101600647,
      "rewards/margins": 3.443368911743164,
      "rewards/rejected": -3.256277322769165,
      "step": 85,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.7045570916538658,
      "grad_norm": 4.725898265838623,
      "learning_rate": 0.00015463487845069707,
      "logits/chosen": -0.34689807891845703,
      "logits/rejected": -0.5313670635223389,
      "logps/chosen": -4.447125434875488,
      "logps/rejected": -49.41028594970703,
      "loss": 0.5447558760643005,
      "memory(GiB)": 46.69,
      "nll_loss": 0.36467093229293823,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.10410800576210022,
      "rewards/margins": 3.554682731628418,
      "rewards/rejected": -3.4505743980407715,
      "step": 86,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.7127496159754224,
      "grad_norm": 1.3880412578582764,
      "learning_rate": 0.00015349078513860726,
      "logits/chosen": -0.21217960119247437,
      "logits/rejected": -0.5045182108879089,
      "logps/chosen": -5.141025066375732,
      "logps/rejected": -67.65399169921875,
      "loss": 0.456913024187088,
      "memory(GiB)": 46.69,
      "nll_loss": 0.35237863659858704,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11491703987121582,
      "rewards/margins": 4.958295822143555,
      "rewards/rejected": -4.84337854385376,
      "step": 87,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.720942140296979,
      "grad_norm": 1.8002839088439941,
      "learning_rate": 0.00015233679836966122,
      "logits/chosen": -0.24154867231845856,
      "logits/rejected": -0.5062354207038879,
      "logps/chosen": -5.772841930389404,
      "logps/rejected": -66.82902526855469,
      "loss": 0.4451909363269806,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3502950370311737,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.15018467605113983,
      "rewards/margins": 5.191023349761963,
      "rewards/rejected": -5.040838718414307,
      "step": 88,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.7291346646185356,
      "grad_norm": 9.176179885864258,
      "learning_rate": 0.0001511731315809637,
      "logits/chosen": -0.25781068205833435,
      "logits/rejected": -0.5210597515106201,
      "logps/chosen": -4.974169731140137,
      "logps/rejected": -70.27227020263672,
      "loss": 0.5398030877113342,
      "memory(GiB)": 46.69,
      "nll_loss": 0.43606072664260864,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.06586135923862457,
      "rewards/margins": 5.657458782196045,
      "rewards/rejected": -5.591597557067871,
      "step": 89,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.7373271889400922,
      "grad_norm": 3.2724390029907227,
      "learning_rate": 0.00015000000000000001,
      "logits/chosen": -0.28262859582901,
      "logits/rejected": -0.47202375531196594,
      "logps/chosen": -7.422355651855469,
      "logps/rejected": -65.22004699707031,
      "loss": 0.5515369176864624,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4296312928199768,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.16720566153526306,
      "rewards/margins": 5.253065586090088,
      "rewards/rejected": -5.085859775543213,
      "step": 90,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.7455197132616488,
      "grad_norm": 3.3079111576080322,
      "learning_rate": 0.00014881762060482814,
      "logits/chosen": -0.3508756756782532,
      "logits/rejected": -0.5413344502449036,
      "logps/chosen": -7.08647346496582,
      "logps/rejected": -65.4574203491211,
      "loss": 0.6241497993469238,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5051572322845459,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": -0.018640263006091118,
      "rewards/margins": 5.245330333709717,
      "rewards/rejected": -5.263970851898193,
      "step": 91,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.7537122375832054,
      "grad_norm": 4.5375657081604,
      "learning_rate": 0.0001476262120839475,
      "logits/chosen": -0.25084343552589417,
      "logits/rejected": -0.5220094919204712,
      "logps/chosen": -6.930018424987793,
      "logps/rejected": -68.6851806640625,
      "loss": 0.6829227805137634,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4883330166339874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04491741210222244,
      "rewards/margins": 5.477269172668457,
      "rewards/rejected": -5.432352066040039,
      "step": 92,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.7619047619047619,
      "grad_norm": 4.925400733947754,
      "learning_rate": 0.00014642599479585105,
      "logits/chosen": -0.21066614985466003,
      "logits/rejected": -0.4863825738430023,
      "logps/chosen": -5.8146772384643555,
      "logps/rejected": -66.52094268798828,
      "loss": 0.5426443815231323,
      "memory(GiB)": 46.69,
      "nll_loss": 0.39929336309432983,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19084298610687256,
      "rewards/margins": 5.132606029510498,
      "rewards/rejected": -4.941762924194336,
      "step": 93,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.7700972862263185,
      "grad_norm": 19.941701889038086,
      "learning_rate": 0.00014521719072826858,
      "logits/chosen": -0.20066988468170166,
      "logits/rejected": -0.4895756244659424,
      "logps/chosen": -7.3873491287231445,
      "logps/rejected": -68.1123046875,
      "loss": 0.9009978771209717,
      "memory(GiB)": 46.69,
      "nll_loss": 0.6462347507476807,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.08460697531700134,
      "rewards/margins": 5.175359725952148,
      "rewards/rejected": -5.259966850280762,
      "step": 94,
      "train_speed(iter/s)": 0.005599
    },
    {
      "epoch": 0.778289810547875,
      "grad_norm": 5.666788578033447,
      "learning_rate": 0.0001440000234571087,
      "logits/chosen": -0.2780590355396271,
      "logits/rejected": -0.5297070145606995,
      "logps/chosen": -6.668648719787598,
      "logps/rejected": -65.53746795654297,
      "loss": 0.6691681146621704,
      "memory(GiB)": 46.69,
      "nll_loss": 0.5282520055770874,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04516776278614998,
      "rewards/margins": 5.023134231567383,
      "rewards/rejected": -5.068301677703857,
      "step": 95,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.7864823348694316,
      "grad_norm": 1.9480782747268677,
      "learning_rate": 0.0001427747181051071,
      "logits/chosen": -0.2837526500225067,
      "logits/rejected": -0.5213181972503662,
      "logps/chosen": -4.880465984344482,
      "logps/rejected": -66.81868743896484,
      "loss": 0.48239535093307495,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3717819154262543,
      "rewards/accuracies": 0.984375,
      "rewards/chosen": 0.11854566633701324,
      "rewards/margins": 5.176722049713135,
      "rewards/rejected": -5.058176517486572,
      "step": 96,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.7946748591909882,
      "grad_norm": 8.093029975891113,
      "learning_rate": 0.00014154150130018866,
      "logits/chosen": -0.3176836371421814,
      "logits/rejected": -0.4916881322860718,
      "logps/chosen": -6.238916397094727,
      "logps/rejected": -63.57952117919922,
      "loss": 0.5672283172607422,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4339982867240906,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14566771686077118,
      "rewards/margins": 5.059505462646484,
      "rewards/rejected": -4.913837432861328,
      "step": 97,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.8028673835125448,
      "grad_norm": 1.8274015188217163,
      "learning_rate": 0.00014030060113355116,
      "logits/chosen": -0.22647900879383087,
      "logits/rejected": -0.5299174189567566,
      "logps/chosen": -4.703901767730713,
      "logps/rejected": -76.23712158203125,
      "loss": 0.6296254992485046,
      "memory(GiB)": 46.69,
      "nll_loss": 0.4894722104072571,
      "rewards/accuracies": 0.953125,
      "rewards/chosen": 0.11238622665405273,
      "rewards/margins": 6.080066204071045,
      "rewards/rejected": -5.967679977416992,
      "step": 98,
      "train_speed(iter/s)": 0.005601
    },
    {
      "epoch": 0.8110599078341014,
      "grad_norm": 4.079970836639404,
      "learning_rate": 0.00013905224711747844,
      "logits/chosen": -0.1417064517736435,
      "logits/rejected": -0.4025101661682129,
      "logps/chosen": -5.256982803344727,
      "logps/rejected": -58.6949462890625,
      "loss": 0.5856602787971497,
      "memory(GiB)": 46.69,
      "nll_loss": 0.3981451094150543,
      "rewards/accuracies": 0.921875,
      "rewards/chosen": 0.1413937211036682,
      "rewards/margins": 4.344367980957031,
      "rewards/rejected": -4.202974319458008,
      "step": 99,
      "train_speed(iter/s)": 0.0056
    },
    {
      "epoch": 0.819252432155658,
      "grad_norm": 2.7077598571777344,
      "learning_rate": 0.00013779667014289065,
      "logits/chosen": -0.27587300539016724,
      "logits/rejected": -0.5051173567771912,
      "logps/chosen": -6.017627239227295,
      "logps/rejected": -57.81541442871094,
      "loss": 0.6035357117652893,
      "memory(GiB)": 46.69,
      "nll_loss": 0.42041608691215515,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14976347982883453,
      "rewards/margins": 4.57808256149292,
      "rewards/rejected": -4.428318977355957,
      "step": 100,
      "train_speed(iter/s)": 0.005601
    }
  ],
  "logging_steps": 1,
  "max_steps": 244,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.1944631776313344e+17,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}