{
  "best_global_step": 300,
  "best_metric": 0.77076083,
  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task3_dpo_gt6_task2_gt60.5+task3gt5/v1-20250609-010021/checkpoint-300",
  "epoch": 0.5980030968017513,
  "eval_steps": 300,
  "global_step": 350,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0017085802765764322,
      "grad_norm": 9.203307151794434,
      "learning_rate": 3.3898305084745763e-06,
      "logits/chosen": -0.6409764885902405,
      "logits/rejected": -0.6413162350654602,
      "logps/chosen": -8.591513633728027,
      "logps/rejected": -9.922354698181152,
      "loss": 1.2514135837554932,
      "memory(GiB)": 45.3,
      "nll_loss": 0.5582665801048279,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.005423
    },
    {
      "epoch": 0.0034171605531528645,
      "grad_norm": 4.572653293609619,
      "learning_rate": 6.779661016949153e-06,
      "logits/chosen": -0.6253237724304199,
      "logits/rejected": -0.6232100129127502,
      "logps/chosen": -9.215431213378906,
      "logps/rejected": -6.302733898162842,
      "loss": 1.1804862022399902,
      "memory(GiB)": 45.3,
      "nll_loss": 0.4873391091823578,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.005426
    },
    {
      "epoch": 0.005125740829729297,
      "grad_norm": 10.135663986206055,
      "learning_rate": 1.016949152542373e-05,
      "logits/chosen": -0.5830379724502563,
      "logits/rejected": -0.5847949981689453,
      "logps/chosen": -6.387916088104248,
      "logps/rejected": -12.526434898376465,
      "loss": 1.1388403177261353,
      "memory(GiB)": 46.08,
      "nll_loss": 0.44991743564605713,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0018886500038206577,
      "rewards/margins": 0.008566470816731453,
      "rewards/rejected": -0.006677820347249508,
      "step": 3,
      "train_speed(iter/s)": 0.005372
    },
    {
      "epoch": 0.006834321106305729,
      "grad_norm": 8.09815788269043,
      "learning_rate": 1.3559322033898305e-05,
      "logits/chosen": -0.5831664800643921,
      "logits/rejected": -0.5881690382957458,
      "logps/chosen": -9.77461051940918,
      "logps/rejected": -15.103912353515625,
      "loss": 1.2644962072372437,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5699671506881714,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0013185496209189296,
      "rewards/margins": -0.0025646828580647707,
      "rewards/rejected": 0.0012461334699764848,
      "step": 4,
      "train_speed(iter/s)": 0.005433
    },
    {
      "epoch": 0.008542901382882162,
      "grad_norm": 10.546202659606934,
      "learning_rate": 1.694915254237288e-05,
      "logits/chosen": -0.6210501194000244,
      "logits/rejected": -0.6261105537414551,
      "logps/chosen": -9.743502616882324,
      "logps/rejected": -13.954730987548828,
      "loss": 1.2041635513305664,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5165743231773376,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.014782106503844261,
      "rewards/margins": 0.011779396794736385,
      "rewards/rejected": 0.0030027113389223814,
      "step": 5,
      "train_speed(iter/s)": 0.005434
    },
    {
      "epoch": 0.010251481659458593,
      "grad_norm": 5.953731060028076,
      "learning_rate": 2.033898305084746e-05,
      "logits/chosen": -0.6194843649864197,
      "logits/rejected": -0.6225060224533081,
      "logps/chosen": -8.993276596069336,
      "logps/rejected": -11.800742149353027,
      "loss": 1.2383989095687866,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5413038730621338,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.02151685394346714,
      "rewards/margins": -0.004275719169527292,
      "rewards/rejected": 0.025792576372623444,
      "step": 6,
      "train_speed(iter/s)": 0.005421
    },
    {
      "epoch": 0.011960061936035027,
      "grad_norm": 3.4447383880615234,
      "learning_rate": 2.3728813559322036e-05,
      "logits/chosen": -0.5979399681091309,
      "logits/rejected": -0.597865104675293,
      "logps/chosen": -11.88587760925293,
      "logps/rejected": -12.513296127319336,
      "loss": 1.1787316799163818,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4771195948123932,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.001614744309335947,
      "rewards/margins": -0.006370083428919315,
      "rewards/rejected": 0.007984823547303677,
      "step": 7,
      "train_speed(iter/s)": 0.005421
    },
    {
      "epoch": 0.013668642212611458,
      "grad_norm": 4.3422627449035645,
      "learning_rate": 2.711864406779661e-05,
      "logits/chosen": -0.5695681571960449,
      "logits/rejected": -0.5736703276634216,
      "logps/chosen": -8.691235542297363,
      "logps/rejected": -17.510940551757812,
      "loss": 1.085353970527649,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4569101631641388,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.061018284410238266,
      "rewards/margins": 0.16366331279277802,
      "rewards/rejected": -0.10264502465724945,
      "step": 8,
      "train_speed(iter/s)": 0.005421
    },
    {
      "epoch": 0.015377222489187891,
      "grad_norm": 6.347766399383545,
      "learning_rate": 3.050847457627119e-05,
      "logits/chosen": -0.6150330305099487,
      "logits/rejected": -0.6163787841796875,
      "logps/chosen": -14.26973819732666,
      "logps/rejected": -12.656587600708008,
      "loss": 1.3125040531158447,
      "memory(GiB)": 46.08,
      "nll_loss": 0.6051884293556213,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06490564346313477,
      "rewards/margins": 0.03234899416565895,
      "rewards/rejected": -0.09725463390350342,
      "step": 9,
      "train_speed(iter/s)": 0.005427
    },
    {
      "epoch": 0.017085802765764324,
      "grad_norm": 3.609342575073242,
      "learning_rate": 3.389830508474576e-05,
      "logits/chosen": -0.5823943018913269,
      "logits/rejected": -0.5893073678016663,
      "logps/chosen": -12.539135932922363,
      "logps/rejected": -19.231651306152344,
      "loss": 1.154615044593811,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5292682647705078,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0102485166862607,
      "rewards/margins": 0.24539220333099365,
      "rewards/rejected": -0.2556407153606415,
      "step": 10,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 0.018794383042340754,
      "grad_norm": 4.803543567657471,
      "learning_rate": 3.728813559322034e-05,
      "logits/chosen": -0.6298958659172058,
      "logits/rejected": -0.6301961541175842,
      "logps/chosen": -10.286799430847168,
      "logps/rejected": -11.364641189575195,
      "loss": 1.143830418586731,
      "memory(GiB)": 46.08,
      "nll_loss": 0.43475160002708435,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06166127696633339,
      "rewards/margins": 0.1280723363161087,
      "rewards/rejected": -0.189733624458313,
      "step": 11,
      "train_speed(iter/s)": 0.005459
    },
    {
      "epoch": 0.020502963318917187,
      "grad_norm": 6.843598365783691,
      "learning_rate": 4.067796610169492e-05,
      "logits/chosen": -0.6147834062576294,
      "logits/rejected": -0.6201154589653015,
      "logps/chosen": -11.104015350341797,
      "logps/rejected": -14.71792221069336,
      "loss": 1.2689580917358398,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4583723843097687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1446399688720703,
      "rewards/margins": -0.06440749764442444,
      "rewards/rejected": -0.08023246377706528,
      "step": 12,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.02221154359549362,
      "grad_norm": 4.361201286315918,
      "learning_rate": 4.4067796610169495e-05,
      "logits/chosen": -0.6593450307846069,
      "logits/rejected": -0.6623726487159729,
      "logps/chosen": -8.788830757141113,
      "logps/rejected": -12.09660530090332,
      "loss": 1.091460108757019,
      "memory(GiB)": 46.08,
      "nll_loss": 0.38157719373703003,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.03397805988788605,
      "rewards/margins": 0.08494514226913452,
      "rewards/rejected": -0.11892320215702057,
      "step": 13,
      "train_speed(iter/s)": 0.005465
    },
    {
      "epoch": 0.023920123872070053,
      "grad_norm": 4.36707067489624,
      "learning_rate": 4.745762711864407e-05,
      "logits/chosen": -0.6003404259681702,
      "logits/rejected": -0.6018444895744324,
      "logps/chosen": -9.472915649414062,
      "logps/rejected": -12.008777618408203,
      "loss": 1.2156641483306885,
      "memory(GiB)": 46.08,
      "nll_loss": 0.42055219411849976,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.07689498364925385,
      "rewards/margins": -0.1264859437942505,
      "rewards/rejected": 0.049590952694416046,
      "step": 14,
      "train_speed(iter/s)": 0.00548
    },
    {
      "epoch": 0.025628704148646483,
      "grad_norm": 3.2864296436309814,
      "learning_rate": 5.0847457627118643e-05,
      "logits/chosen": -0.6285042762756348,
      "logits/rejected": -0.6272621154785156,
      "logps/chosen": -14.201019287109375,
      "logps/rejected": -10.998590469360352,
      "loss": 1.2311502695083618,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5152750015258789,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06618006527423859,
      "rewards/margins": 0.029741406440734863,
      "rewards/rejected": 0.036438651382923126,
      "step": 15,
      "train_speed(iter/s)": 0.005474
    },
    {
      "epoch": 0.027337284425222916,
      "grad_norm": 2.4144492149353027,
      "learning_rate": 5.423728813559322e-05,
      "logits/chosen": -0.6098039746284485,
      "logits/rejected": -0.6130300760269165,
      "logps/chosen": -3.4820291996002197,
      "logps/rejected": -16.196063995361328,
      "loss": 0.8393359780311584,
      "memory(GiB)": 46.08,
      "nll_loss": 0.214947909116745,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0826798751950264,
      "rewards/margins": 0.17492221295833588,
      "rewards/rejected": -0.09224233776330948,
      "step": 16,
      "train_speed(iter/s)": 0.005477
    },
    {
      "epoch": 0.02904586470179935,
      "grad_norm": 2.3232851028442383,
      "learning_rate": 5.76271186440678e-05,
      "logits/chosen": -0.5909145474433899,
      "logits/rejected": -0.593756377696991,
      "logps/chosen": -7.929438591003418,
      "logps/rejected": -14.363272666931152,
      "loss": 0.9911059737205505,
      "memory(GiB)": 46.08,
      "nll_loss": 0.34508591890335083,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.10390710830688477,
      "rewards/margins": 0.13225752115249634,
      "rewards/rejected": -0.028350409120321274,
      "step": 17,
      "train_speed(iter/s)": 0.005482
    },
    {
      "epoch": 0.030754444978375782,
      "grad_norm": 2.344477891921997,
      "learning_rate": 6.101694915254238e-05,
      "logits/chosen": -0.5697246789932251,
      "logits/rejected": -0.5692474246025085,
      "logps/chosen": -6.287097930908203,
      "logps/rejected": -10.676097869873047,
      "loss": 0.9974570274353027,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3142527639865875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.09520465135574341,
      "rewards/margins": 0.04853628948330879,
      "rewards/rejected": 0.046668365597724915,
      "step": 18,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.03246302525495221,
      "grad_norm": 2.361417531967163,
      "learning_rate": 6.440677966101695e-05,
      "logits/chosen": -0.5909584164619446,
      "logits/rejected": -0.596694827079773,
      "logps/chosen": -4.706894874572754,
      "logps/rejected": -12.18591022491455,
      "loss": 0.8993738889694214,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2403678447008133,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1180216372013092,
      "rewards/margins": 0.09891890734434128,
      "rewards/rejected": 0.01910274103283882,
      "step": 19,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.03417160553152865,
      "grad_norm": 2.1051840782165527,
      "learning_rate": 6.779661016949152e-05,
      "logits/chosen": -0.5925375819206238,
      "logits/rejected": -0.5939764380455017,
      "logps/chosen": -7.690135478973389,
      "logps/rejected": -11.360199928283691,
      "loss": 0.9811086058616638,
      "memory(GiB)": 46.08,
      "nll_loss": 0.27671271562576294,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.15354080498218536,
      "rewards/margins": 0.02018781006336212,
      "rewards/rejected": 0.13335299491882324,
      "step": 20,
      "train_speed(iter/s)": 0.005507
    },
    {
      "epoch": 0.03588018580810508,
      "grad_norm": 2.453577995300293,
      "learning_rate": 7.11864406779661e-05,
      "logits/chosen": -0.589929461479187,
      "logits/rejected": -0.5913950800895691,
      "logps/chosen": -6.006174564361572,
      "logps/rejected": -12.356660842895508,
      "loss": 0.8762722015380859,
      "memory(GiB)": 46.08,
      "nll_loss": 0.26102662086486816,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.15477147698402405,
      "rewards/margins": 0.20186343789100647,
      "rewards/rejected": -0.047091953456401825,
      "step": 21,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.03758876608468151,
      "grad_norm": 2.334592819213867,
      "learning_rate": 7.457627118644068e-05,
      "logits/chosen": -0.6286447048187256,
      "logits/rejected": -0.6310911178588867,
      "logps/chosen": -9.828102111816406,
      "logps/rejected": -16.171802520751953,
      "loss": 0.9827708601951599,
      "memory(GiB)": 46.08,
      "nll_loss": 0.40016523003578186,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.22255462408065796,
      "rewards/margins": 0.2890108525753021,
      "rewards/rejected": -0.06645623594522476,
      "step": 22,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.039297346361257944,
      "grad_norm": 2.763946533203125,
      "learning_rate": 7.796610169491526e-05,
      "logits/chosen": -0.6423724889755249,
      "logits/rejected": -0.6475900411605835,
      "logps/chosen": -7.048077583312988,
      "logps/rejected": -17.518115997314453,
      "loss": 1.029776692390442,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4258265495300293,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13568007946014404,
      "rewards/margins": 0.24192118644714355,
      "rewards/rejected": -0.10624107718467712,
      "step": 23,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.041005926637834374,
      "grad_norm": 5.438773155212402,
      "learning_rate": 8.135593220338983e-05,
      "logits/chosen": -0.611075758934021,
      "logits/rejected": -0.6109172701835632,
      "logps/chosen": -8.579011917114258,
      "logps/rejected": -12.74751091003418,
      "loss": 1.2176915407180786,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5213994979858398,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.010570855811238289,
      "rewards/margins": 0.06740333139896393,
      "rewards/rejected": -0.07797417044639587,
      "step": 24,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.0427145069144108,
      "grad_norm": 3.8305866718292236,
      "learning_rate": 8.474576271186441e-05,
      "logits/chosen": -0.6644467711448669,
      "logits/rejected": -0.6674079298973083,
      "logps/chosen": -6.975298881530762,
      "logps/rejected": -15.503819465637207,
      "loss": 0.922063410282135,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3326256275177002,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06792093813419342,
      "rewards/margins": 0.324211061000824,
      "rewards/rejected": -0.25629013776779175,
      "step": 25,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.04442308719098724,
      "grad_norm": 4.446774959564209,
      "learning_rate": 8.813559322033899e-05,
      "logits/chosen": -0.6409133076667786,
      "logits/rejected": -0.6452510356903076,
      "logps/chosen": -5.22960090637207,
      "logps/rejected": -16.918365478515625,
      "loss": 0.7774365544319153,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2448062300682068,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2400483787059784,
      "rewards/margins": 0.39764925837516785,
      "rewards/rejected": -0.15760089457035065,
      "step": 26,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.04613166746756367,
      "grad_norm": 2.5869643688201904,
      "learning_rate": 9.152542372881357e-05,
      "logits/chosen": -0.6091906428337097,
      "logits/rejected": -0.6120930910110474,
      "logps/chosen": -7.03964376449585,
      "logps/rejected": -18.091930389404297,
      "loss": 0.8498358726501465,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3106813132762909,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1571023166179657,
      "rewards/margins": 0.4190843403339386,
      "rewards/rejected": -0.2619820237159729,
      "step": 27,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.047840247744140106,
      "grad_norm": 3.0630600452423096,
      "learning_rate": 9.491525423728815e-05,
      "logits/chosen": -0.6014366745948792,
      "logits/rejected": -0.6016579270362854,
      "logps/chosen": -9.281977653503418,
      "logps/rejected": -13.255182266235352,
      "loss": 0.9999822974205017,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3916056454181671,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0838853195309639,
      "rewards/margins": 0.22357328236103058,
      "rewards/rejected": -0.13968798518180847,
      "step": 28,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.049548828020716536,
      "grad_norm": 4.956308364868164,
      "learning_rate": 9.830508474576272e-05,
      "logits/chosen": -0.6361557841300964,
      "logits/rejected": -0.6375819444656372,
      "logps/chosen": -5.407459735870361,
      "logps/rejected": -12.960749626159668,
      "loss": 0.9380594491958618,
      "memory(GiB)": 46.08,
      "nll_loss": 0.39800193905830383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14713114500045776,
      "rewards/margins": 0.4169777035713196,
      "rewards/rejected": -0.2698465585708618,
      "step": 29,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.051257408297292965,
      "grad_norm": 6.679542541503906,
      "learning_rate": 0.00010169491525423729,
      "logits/chosen": -0.6229231357574463,
      "logits/rejected": -0.6250395178794861,
      "logps/chosen": -11.77493667602539,
      "logps/rejected": -18.192317962646484,
      "loss": 0.7899165749549866,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2795858085155487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09651458263397217,
      "rewards/margins": 0.49087047576904297,
      "rewards/rejected": -0.3943559229373932,
      "step": 30,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.0529659885738694,
      "grad_norm": 4.011873245239258,
      "learning_rate": 0.00010508474576271188,
      "logits/chosen": -0.6524882912635803,
      "logits/rejected": -0.6541539430618286,
      "logps/chosen": -7.8713202476501465,
      "logps/rejected": -11.641451835632324,
      "loss": 0.9657379984855652,
      "memory(GiB)": 46.08,
      "nll_loss": 0.40233710408210754,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.11578395962715149,
      "rewards/margins": 0.36746829748153687,
      "rewards/rejected": -0.25168436765670776,
      "step": 31,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.05467456885044583,
      "grad_norm": 4.144499778747559,
      "learning_rate": 0.00010847457627118644,
      "logits/chosen": -0.6458109617233276,
      "logits/rejected": -0.6519441604614258,
      "logps/chosen": -8.17459487915039,
      "logps/rejected": -19.17450714111328,
      "loss": 0.9243566989898682,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3649522066116333,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.17762523889541626,
      "rewards/margins": 0.41990581154823303,
      "rewards/rejected": -0.24228057265281677,
      "step": 32,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.05638314912702227,
      "grad_norm": 2.5846283435821533,
      "learning_rate": 0.00011186440677966102,
      "logits/chosen": -0.6422857046127319,
      "logits/rejected": -0.64764004945755,
      "logps/chosen": -7.8391828536987305,
      "logps/rejected": -16.905941009521484,
      "loss": 0.8121039867401123,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3524104952812195,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.21112650632858276,
      "rewards/margins": 0.6391255259513855,
      "rewards/rejected": -0.42799898982048035,
      "step": 33,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.0580917294035987,
      "grad_norm": 2.831099271774292,
      "learning_rate": 0.0001152542372881356,
      "logits/chosen": -0.6351624727249146,
      "logits/rejected": -0.6340872645378113,
      "logps/chosen": -8.873239517211914,
      "logps/rejected": -14.13478946685791,
      "loss": 0.9939954280853271,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5031149983406067,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.15203362703323364,
      "rewards/margins": 0.5661364793777466,
      "rewards/rejected": -0.41410285234451294,
      "step": 34,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.05980030968017513,
      "grad_norm": 6.754673004150391,
      "learning_rate": 0.00011864406779661017,
      "logits/chosen": -0.6350009441375732,
      "logits/rejected": -0.6345161199569702,
      "logps/chosen": -9.049372673034668,
      "logps/rejected": -13.872842788696289,
      "loss": 0.9677298069000244,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4823642373085022,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18822304904460907,
      "rewards/margins": 0.5690968632698059,
      "rewards/rejected": -0.38087382912635803,
      "step": 35,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.061508889956751564,
      "grad_norm": 5.09066915512085,
      "learning_rate": 0.00012203389830508477,
      "logits/chosen": -0.5940477252006531,
      "logits/rejected": -0.5995637774467468,
      "logps/chosen": -6.552436351776123,
      "logps/rejected": -22.02668571472168,
      "loss": 1.0198695659637451,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5385282635688782,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.09731997549533844,
      "rewards/margins": 0.7183221578598022,
      "rewards/rejected": -0.621002197265625,
      "step": 36,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.063217470233328,
      "grad_norm": 6.366957664489746,
      "learning_rate": 0.00012542372881355933,
      "logits/chosen": -0.5671308040618896,
      "logits/rejected": -0.5699396133422852,
      "logps/chosen": -8.690061569213867,
      "logps/rejected": -18.227741241455078,
      "loss": 0.9119342565536499,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4206831753253937,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.008592324331402779,
      "rewards/margins": 0.7837947010993958,
      "rewards/rejected": -0.7752023935317993,
      "step": 37,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.06492605050990442,
      "grad_norm": 8.224360466003418,
      "learning_rate": 0.0001288135593220339,
      "logits/chosen": -0.6006415486335754,
      "logits/rejected": -0.6039485931396484,
      "logps/chosen": -7.98246955871582,
      "logps/rejected": -19.214567184448242,
      "loss": 1.020494818687439,
      "memory(GiB)": 46.08,
      "nll_loss": 0.47989213466644287,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.017101695761084557,
      "rewards/margins": 0.7686665654182434,
      "rewards/rejected": -0.7857682704925537,
      "step": 38,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.06663463078648085,
      "grad_norm": 5.284773349761963,
      "learning_rate": 0.00013220338983050849,
      "logits/chosen": -0.5746678709983826,
      "logits/rejected": -0.5742544531822205,
      "logps/chosen": -10.300456047058105,
      "logps/rejected": -14.731122970581055,
      "loss": 1.1951102018356323,
      "memory(GiB)": 46.08,
      "nll_loss": 0.6024460196495056,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005457160994410515,
      "rewards/margins": 0.43640822172164917,
      "rewards/rejected": -0.44186532497406006,
      "step": 39,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.0683432110630573,
      "grad_norm": 4.581506252288818,
      "learning_rate": 0.00013559322033898305,
      "logits/chosen": -0.5436846613883972,
      "logits/rejected": -0.5412294268608093,
      "logps/chosen": -11.306229591369629,
      "logps/rejected": -15.808780670166016,
      "loss": 0.9979494214057922,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4069577753543854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10538371652364731,
      "rewards/margins": 0.4707936942577362,
      "rewards/rejected": -0.5761774778366089,
      "step": 40,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.07005179133963373,
      "grad_norm": 4.280210018157959,
      "learning_rate": 0.00013898305084745764,
      "logits/chosen": -0.5919702649116516,
      "logits/rejected": -0.5944460034370422,
      "logps/chosen": -4.601211071014404,
      "logps/rejected": -18.39055061340332,
      "loss": 0.7289348244667053,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2620496451854706,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1577654778957367,
      "rewards/margins": 0.7140093445777893,
      "rewards/rejected": -0.5562438368797302,
      "step": 41,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.07176037161621016,
      "grad_norm": 2.8965229988098145,
      "learning_rate": 0.0001423728813559322,
      "logits/chosen": -0.5226963758468628,
      "logits/rejected": -0.5210604667663574,
      "logps/chosen": -8.042407035827637,
      "logps/rejected": -13.117356300354004,
      "loss": 0.8382841944694519,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3229953646659851,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.15337654948234558,
      "rewards/margins": 0.5258168578147888,
      "rewards/rejected": -0.3724403381347656,
      "step": 42,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.07346895189278659,
      "grad_norm": 4.7869791984558105,
      "learning_rate": 0.00014576271186440677,
      "logits/chosen": -0.5595600605010986,
      "logits/rejected": -0.5619646310806274,
      "logps/chosen": -5.741379261016846,
      "logps/rejected": -14.867729187011719,
      "loss": 0.9325981140136719,
      "memory(GiB)": 46.08,
      "nll_loss": 0.34299078583717346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06792251765727997,
      "rewards/margins": 0.3079211115837097,
      "rewards/rejected": -0.23999860882759094,
      "step": 43,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.07517753216936301,
      "grad_norm": 3.8402724266052246,
      "learning_rate": 0.00014915254237288136,
      "logits/chosen": -0.5551996231079102,
      "logits/rejected": -0.5583171248435974,
      "logps/chosen": -7.729850769042969,
      "logps/rejected": -14.535001754760742,
      "loss": 0.985005795955658,
      "memory(GiB)": 46.08,
      "nll_loss": 0.37224525213241577,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0475621297955513,
      "rewards/margins": 0.2842516601085663,
      "rewards/rejected": -0.23668958246707916,
      "step": 44,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.07688611244593946,
      "grad_norm": 3.136073112487793,
      "learning_rate": 0.00015254237288135592,
      "logits/chosen": -0.5538796186447144,
      "logits/rejected": -0.5563740730285645,
      "logps/chosen": -8.61514663696289,
      "logps/rejected": -20.434301376342773,
      "loss": 0.8447250723838806,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3852572441101074,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17051486670970917,
      "rewards/margins": 0.695674479007721,
      "rewards/rejected": -0.5251596570014954,
      "step": 45,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.07859469272251589,
      "grad_norm": 2.950076103210449,
      "learning_rate": 0.00015593220338983051,
      "logits/chosen": -0.588573694229126,
      "logits/rejected": -0.5898380875587463,
      "logps/chosen": -8.49618911743164,
      "logps/rejected": -16.47633171081543,
      "loss": 0.8776077032089233,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3434315621852875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1189945787191391,
      "rewards/margins": 0.5393712520599365,
      "rewards/rejected": -0.42037665843963623,
      "step": 46,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.08030327299909232,
      "grad_norm": 2.918926954269409,
      "learning_rate": 0.00015932203389830508,
      "logits/chosen": -0.5735058188438416,
      "logits/rejected": -0.5742441415786743,
      "logps/chosen": -10.007829666137695,
      "logps/rejected": -15.225828170776367,
      "loss": 0.9683640003204346,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4518345892429352,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.29521822929382324,
      "rewards/margins": 0.5080306529998779,
      "rewards/rejected": -0.2128123939037323,
      "step": 47,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.08201185327566875,
      "grad_norm": 2.885613441467285,
      "learning_rate": 0.00016271186440677967,
      "logits/chosen": -0.5690568089485168,
      "logits/rejected": -0.570725679397583,
      "logps/chosen": -6.9214887619018555,
      "logps/rejected": -17.073692321777344,
      "loss": 0.8013411164283752,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2747730612754822,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1803244799375534,
      "rewards/margins": 0.5454539060592651,
      "rewards/rejected": -0.36512941122055054,
      "step": 48,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.08372043355224518,
      "grad_norm": 2.8386495113372803,
      "learning_rate": 0.00016610169491525423,
      "logits/chosen": -0.5691187381744385,
      "logits/rejected": -0.5723721981048584,
      "logps/chosen": -6.647555828094482,
      "logps/rejected": -18.68927764892578,
      "loss": 0.731940746307373,
      "memory(GiB)": 46.08,
      "nll_loss": 0.31857162714004517,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15964265167713165,
      "rewards/margins": 0.8172457218170166,
      "rewards/rejected": -0.6576029062271118,
      "step": 49,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.0854290138288216,
      "grad_norm": 6.678228855133057,
      "learning_rate": 0.00016949152542372882,
      "logits/chosen": -0.569486141204834,
      "logits/rejected": -0.5724078416824341,
      "logps/chosen": -7.978659629821777,
      "logps/rejected": -18.0915584564209,
      "loss": 0.8816509246826172,
      "memory(GiB)": 46.08,
      "nll_loss": 0.41086509823799133,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16925573348999023,
      "rewards/margins": 0.6901466846466064,
      "rewards/rejected": -0.520891010761261,
      "step": 50,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.08713759410539805,
      "grad_norm": 3.4269354343414307,
      "learning_rate": 0.00017288135593220342,
      "logits/chosen": -0.6229482889175415,
      "logits/rejected": -0.624045193195343,
      "logps/chosen": -8.981767654418945,
      "logps/rejected": -18.87827491760254,
      "loss": 0.902818500995636,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4753139615058899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12949834764003754,
      "rewards/margins": 0.91382896900177,
      "rewards/rejected": -0.7843306064605713,
      "step": 51,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.08884617438197448,
      "grad_norm": 3.730417013168335,
      "learning_rate": 0.00017627118644067798,
      "logits/chosen": -0.5907954573631287,
      "logits/rejected": -0.5928065776824951,
      "logps/chosen": -8.826777458190918,
      "logps/rejected": -25.743032455444336,
      "loss": 0.8241711854934692,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4183700382709503,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15070247650146484,
      "rewards/margins": 1.0849279165267944,
      "rewards/rejected": -0.9342254400253296,
      "step": 52,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.09055475465855091,
      "grad_norm": 6.543994426727295,
      "learning_rate": 0.00017966101694915257,
      "logits/chosen": -0.5657305717468262,
      "logits/rejected": -0.5669429898262024,
      "logps/chosen": -11.976017951965332,
      "logps/rejected": -18.324581146240234,
      "loss": 1.1098811626434326,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5012481808662415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04995343089103699,
      "rewards/margins": 0.5761421918869019,
      "rewards/rejected": -0.6260955929756165,
      "step": 53,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.09226333493512734,
      "grad_norm": 7.451931476593018,
      "learning_rate": 0.00018305084745762714,
      "logits/chosen": -0.5736872553825378,
      "logits/rejected": -0.5727863907814026,
      "logps/chosen": -9.956096649169922,
      "logps/rejected": -17.354209899902344,
      "loss": 1.0036059617996216,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35273733735084534,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07980930060148239,
      "rewards/margins": 0.5828754901885986,
      "rewards/rejected": -0.6626847982406616,
      "step": 54,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.09397191521170377,
      "grad_norm": 4.901066303253174,
      "learning_rate": 0.0001864406779661017,
      "logits/chosen": -0.5865222811698914,
      "logits/rejected": -0.5857770442962646,
      "logps/chosen": -6.749457359313965,
      "logps/rejected": -19.966379165649414,
      "loss": 0.836351215839386,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30449604988098145,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11927387118339539,
      "rewards/margins": 0.7463572025299072,
      "rewards/rejected": -0.6270833611488342,
      "step": 55,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.09568049548828021,
      "grad_norm": 3.0312962532043457,
      "learning_rate": 0.0001898305084745763,
      "logits/chosen": -0.5644817352294922,
      "logits/rejected": -0.5671055912971497,
      "logps/chosen": -6.525354862213135,
      "logps/rejected": -18.556304931640625,
      "loss": 0.7299084663391113,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2566152811050415,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.051953963935375214,
      "rewards/margins": 0.8131421804428101,
      "rewards/rejected": -0.7611882090568542,
      "step": 56,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.09738907576485664,
      "grad_norm": 6.161406517028809,
      "learning_rate": 0.00019322033898305085,
      "logits/chosen": -0.5638913512229919,
      "logits/rejected": -0.5684786438941956,
      "logps/chosen": -8.483434677124023,
      "logps/rejected": -20.397701263427734,
      "loss": 0.988010585308075,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4928280711174011,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.17585664987564087,
      "rewards/margins": 0.6749651432037354,
      "rewards/rejected": -0.4991084933280945,
      "step": 57,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.09909765604143307,
      "grad_norm": 4.03005313873291,
      "learning_rate": 0.00019661016949152545,
      "logits/chosen": -0.5518775582313538,
      "logits/rejected": -0.552839994430542,
      "logps/chosen": -6.419961929321289,
      "logps/rejected": -20.10483169555664,
      "loss": 0.7270359396934509,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2987552881240845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23015271127223969,
      "rewards/margins": 0.8427435159683228,
      "rewards/rejected": -0.6125907897949219,
      "step": 58,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.1008062363180095,
      "grad_norm": 2.9903881549835205,
      "learning_rate": 0.0002,
      "logits/chosen": -0.558972954750061,
      "logits/rejected": -0.559365451335907,
      "logps/chosen": -7.266337871551514,
      "logps/rejected": -19.700899124145508,
      "loss": 0.8035504221916199,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35604721307754517,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23076069355010986,
      "rewards/margins": 0.6581640243530273,
      "rewards/rejected": -0.4274032711982727,
      "step": 59,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.10251481659458593,
      "grad_norm": 2.308953046798706,
      "learning_rate": 0.00019999960020133237,
      "logits/chosen": -0.5721818804740906,
      "logits/rejected": -0.5736841559410095,
      "logps/chosen": -5.192386150360107,
      "logps/rejected": -16.873666763305664,
      "loss": 0.6971458196640015,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25499823689460754,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.14774081110954285,
      "rewards/margins": 0.7478816509246826,
      "rewards/rejected": -0.6001408696174622,
      "step": 60,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.10422339687116237,
      "grad_norm": 6.15336275100708,
      "learning_rate": 0.00019999840080852627,
      "logits/chosen": -0.6088159680366516,
      "logits/rejected": -0.6091991662979126,
      "logps/chosen": -10.085458755493164,
      "logps/rejected": -16.714794158935547,
      "loss": 1.2082210779190063,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6319302320480347,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.015833575278520584,
      "rewards/margins": 0.46795743703842163,
      "rewards/rejected": -0.4837909936904907,
      "step": 61,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.1059319771477388,
      "grad_norm": 4.543377876281738,
      "learning_rate": 0.00019999640183117196,
      "logits/chosen": -0.6160375475883484,
      "logits/rejected": -0.6183785796165466,
      "logps/chosen": -7.354756832122803,
      "logps/rejected": -17.332752227783203,
      "loss": 0.9280235767364502,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39269304275512695,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.00012398138642311096,
      "rewards/margins": 0.7065204381942749,
      "rewards/rejected": -0.7063965797424316,
      "step": 62,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.10764055742431523,
      "grad_norm": 3.700834035873413,
      "learning_rate": 0.00019999360328525325,
      "logits/chosen": -0.5955262780189514,
      "logits/rejected": -0.5960131287574768,
      "logps/chosen": -7.038027763366699,
      "logps/rejected": -19.62297248840332,
      "loss": 0.8426406383514404,
      "memory(GiB)": 46.38,
      "nll_loss": 0.304790735244751,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04702156037092209,
      "rewards/margins": 0.6756925582885742,
      "rewards/rejected": -0.6286709904670715,
      "step": 63,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.10934913770089166,
      "grad_norm": 3.616783857345581,
      "learning_rate": 0.00019999000519314722,
      "logits/chosen": -0.5933761596679688,
      "logits/rejected": -0.5956672430038452,
      "logps/chosen": -10.029403686523438,
      "logps/rejected": -19.743268966674805,
      "loss": 0.8910276889801025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4178580045700073,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.059932250529527664,
      "rewards/margins": 0.7701122164726257,
      "rewards/rejected": -0.7101800441741943,
      "step": 64,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.11105771797746809,
      "grad_norm": 2.8669040203094482,
      "learning_rate": 0.00019998560758362416,
      "logits/chosen": -0.6144078373908997,
      "logits/rejected": -0.6167560815811157,
      "logps/chosen": -10.685945510864258,
      "logps/rejected": -23.724315643310547,
      "loss": 0.7696695923805237,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3780427873134613,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1446903944015503,
      "rewards/margins": 1.0186232328414917,
      "rewards/rejected": -0.873932957649231,
      "step": 65,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.11276629825404454,
      "grad_norm": 4.312869548797607,
      "learning_rate": 0.00019998041049184719,
      "logits/chosen": -0.620762050151825,
      "logits/rejected": -0.6244264245033264,
      "logps/chosen": -11.728214263916016,
      "logps/rejected": -26.204395294189453,
      "loss": 0.7992648482322693,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4216463565826416,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.14800474047660828,
      "rewards/margins": 1.0791853666305542,
      "rewards/rejected": -0.9311806559562683,
      "step": 66,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.11447487853062097,
      "grad_norm": 7.865874290466309,
      "learning_rate": 0.00019997441395937213,
      "logits/chosen": -0.6298569440841675,
      "logits/rejected": -0.6332789659500122,
      "logps/chosen": -7.128231525421143,
      "logps/rejected": -22.970563888549805,
      "loss": 0.7514320015907288,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33478716015815735,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05204010754823685,
      "rewards/margins": 0.9836788773536682,
      "rewards/rejected": -0.931638777256012,
      "step": 67,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.1161834588071974,
      "grad_norm": 6.099157810211182,
      "learning_rate": 0.0001999676180341471,
      "logits/chosen": -0.6536255478858948,
      "logits/rejected": -0.6564731597900391,
      "logps/chosen": -10.524028778076172,
      "logps/rejected": -19.91427993774414,
      "loss": 1.1307811737060547,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7356545925140381,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.028077581897377968,
      "rewards/margins": 0.9777989387512207,
      "rewards/rejected": -0.9497213363647461,
      "step": 68,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.11789203908377383,
      "grad_norm": 5.27453088760376,
      "learning_rate": 0.00019996002277051218,
      "logits/chosen": -0.6637449264526367,
      "logits/rejected": -0.6651371717453003,
      "logps/chosen": -9.458850860595703,
      "logps/rejected": -16.127042770385742,
      "loss": 1.4500970840454102,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8368474841117859,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.15293718874454498,
      "rewards/margins": 0.500515878200531,
      "rewards/rejected": -0.6534531116485596,
      "step": 69,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.11960061936035025,
      "grad_norm": 4.469899654388428,
      "learning_rate": 0.00019995162822919883,
      "logits/chosen": -0.6642380356788635,
      "logits/rejected": -0.664824366569519,
      "logps/chosen": -8.725358963012695,
      "logps/rejected": -16.164390563964844,
      "loss": 0.8428195714950562,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3803461194038391,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.08946336805820465,
      "rewards/margins": 0.7338303327560425,
      "rewards/rejected": -0.644366979598999,
      "step": 70,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.12130919963692668,
      "grad_norm": 4.995606422424316,
      "learning_rate": 0.00019994243447732957,
      "logits/chosen": -0.6431492567062378,
      "logits/rejected": -0.6473235487937927,
      "logps/chosen": -8.270989418029785,
      "logps/rejected": -24.483354568481445,
      "loss": 0.8698587417602539,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4563717544078827,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07262411713600159,
      "rewards/margins": 1.0772123336791992,
      "rewards/rejected": -1.0045881271362305,
      "step": 71,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.12301777991350313,
      "grad_norm": 3.5250916481018066,
      "learning_rate": 0.00019993244158841747,
      "logits/chosen": -0.6772405505180359,
      "logits/rejected": -0.6789288520812988,
      "logps/chosen": -9.19353199005127,
      "logps/rejected": -27.746652603149414,
      "loss": 0.8238008618354797,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39661213755607605,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09302112460136414,
      "rewards/margins": 1.367937684059143,
      "rewards/rejected": -1.274916410446167,
      "step": 72,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.12472636019007956,
      "grad_norm": 3.276311159133911,
      "learning_rate": 0.00019992164964236533,
      "logits/chosen": -0.6857064962387085,
      "logits/rejected": -0.685559868812561,
      "logps/chosen": -11.672329902648926,
      "logps/rejected": -21.510480880737305,
      "loss": 0.9018468856811523,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41466090083122253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07735104858875275,
      "rewards/margins": 0.9590628147125244,
      "rewards/rejected": -0.8817118406295776,
      "step": 73,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.126434940466656,
      "grad_norm": 3.369281768798828,
      "learning_rate": 0.00019991005872546527,
      "logits/chosen": -0.6608355641365051,
      "logits/rejected": -0.6626392602920532,
      "logps/chosen": -7.696371078491211,
      "logps/rejected": -16.016326904296875,
      "loss": 0.8347861766815186,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3552168607711792,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13171803951263428,
      "rewards/margins": 0.7066640853881836,
      "rewards/rejected": -0.5749460458755493,
      "step": 74,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.12814352074323243,
      "grad_norm": 2.894788980484009,
      "learning_rate": 0.00019989766893039804,
      "logits/chosen": -0.6222079992294312,
      "logits/rejected": -0.6232355833053589,
      "logps/chosen": -5.741198539733887,
      "logps/rejected": -21.31194305419922,
      "loss": 0.7077224850654602,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2760768234729767,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08473706245422363,
      "rewards/margins": 0.9282934665679932,
      "rewards/rejected": -0.8435564041137695,
      "step": 75,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.12985210101980885,
      "grad_norm": 2.1586077213287354,
      "learning_rate": 0.00019988448035623203,
      "logits/chosen": -0.6409361958503723,
      "logits/rejected": -0.6418861150741577,
      "logps/chosen": -7.263876438140869,
      "logps/rejected": -17.95524024963379,
      "loss": 0.8056262731552124,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2723458409309387,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.10251796990633011,
      "rewards/margins": 0.5812361240386963,
      "rewards/rejected": -0.4787181317806244,
      "step": 76,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1315606812963853,
      "grad_norm": 3.0591204166412354,
      "learning_rate": 0.00019987049310842272,
      "logits/chosen": -0.6324824690818787,
      "logits/rejected": -0.6336201429367065,
      "logps/chosen": -10.325702667236328,
      "logps/rejected": -15.849100112915039,
      "loss": 0.8897943496704102,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3855420649051666,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16009055078029633,
      "rewards/margins": 0.6010256409645081,
      "rewards/rejected": -0.4409351050853729,
      "step": 77,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1332692615729617,
      "grad_norm": 2.668419122695923,
      "learning_rate": 0.00019985570729881184,
      "logits/chosen": -0.6462000012397766,
      "logits/rejected": -0.6490585207939148,
      "logps/chosen": -5.475035667419434,
      "logps/rejected": -20.422216415405273,
      "loss": 0.7187508344650269,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26306530833244324,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.13646253943443298,
      "rewards/margins": 0.6754783391952515,
      "rewards/rejected": -0.5390157699584961,
      "step": 78,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.13497784184953815,
      "grad_norm": 4.360518932342529,
      "learning_rate": 0.00019984012304562622,
      "logits/chosen": -0.6780133247375488,
      "logits/rejected": -0.6758167147636414,
      "logps/chosen": -11.310664176940918,
      "logps/rejected": -15.12096881866455,
      "loss": 0.9152082800865173,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45073550939559937,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15347006916999817,
      "rewards/margins": 0.7282993197441101,
      "rewards/rejected": -0.5748292207717896,
      "step": 79,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.1366864221261146,
      "grad_norm": 2.7597768306732178,
      "learning_rate": 0.00019982374047347723,
      "logits/chosen": -0.6909894943237305,
      "logits/rejected": -0.694047212600708,
      "logps/chosen": -6.004232406616211,
      "logps/rejected": -23.221824645996094,
      "loss": 0.6792643666267395,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31354260444641113,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21520498394966125,
      "rewards/margins": 1.1965129375457764,
      "rewards/rejected": -0.9813080430030823,
      "step": 80,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.138395002402691,
      "grad_norm": 4.00593900680542,
      "learning_rate": 0.00019980655971335945,
      "logits/chosen": -0.666761577129364,
      "logits/rejected": -0.6698122620582581,
      "logps/chosen": -11.200495719909668,
      "logps/rejected": -20.89803123474121,
      "loss": 1.0373343229293823,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5717941522598267,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.03891691192984581,
      "rewards/margins": 0.8691489696502686,
      "rewards/rejected": -0.9080657958984375,
      "step": 81,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.14010358267926745,
      "grad_norm": 3.7606523036956787,
      "learning_rate": 0.00019978858090264975,
      "logits/chosen": -0.6808147430419922,
      "logits/rejected": -0.683088481426239,
      "logps/chosen": -8.414874076843262,
      "logps/rejected": -28.505592346191406,
      "loss": 0.7084274888038635,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3828541338443756,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1215677559375763,
      "rewards/margins": 1.7334877252578735,
      "rewards/rejected": -1.6119199991226196,
      "step": 82,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.14181216295584387,
      "grad_norm": 4.108179569244385,
      "learning_rate": 0.0001997698041851063,
      "logits/chosen": -0.6897909045219421,
      "logits/rejected": -0.6913143396377563,
      "logps/chosen": -4.813051223754883,
      "logps/rejected": -26.06230926513672,
      "loss": 0.572483241558075,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24444976449012756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15832601487636566,
      "rewards/margins": 1.6045297384262085,
      "rewards/rejected": -1.4462038278579712,
      "step": 83,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1435207432324203,
      "grad_norm": 5.49106502532959,
      "learning_rate": 0.00019975022971086714,
      "logits/chosen": -0.6466387510299683,
      "logits/rejected": -0.6520243883132935,
      "logps/chosen": -7.595500946044922,
      "logps/rejected": -33.218482971191406,
      "loss": 0.9090702533721924,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5250993371009827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13341139256954193,
      "rewards/margins": 1.5776695013046265,
      "rewards/rejected": -1.444258213043213,
      "step": 84,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.14522932350899675,
      "grad_norm": 4.539761543273926,
      "learning_rate": 0.00019972985763644932,
      "logits/chosen": -0.6421160101890564,
      "logits/rejected": -0.6411360502243042,
      "logps/chosen": -7.4049482345581055,
      "logps/rejected": -21.37047576904297,
      "loss": 0.8407540917396545,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4470757246017456,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1697394847869873,
      "rewards/margins": 1.345821738243103,
      "rewards/rejected": -1.1760823726654053,
      "step": 85,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.14693790378557317,
      "grad_norm": 4.877723217010498,
      "learning_rate": 0.00019970868812474736,
      "logits/chosen": -0.6163697242736816,
      "logits/rejected": -0.6189360022544861,
      "logps/chosen": -12.07491683959961,
      "logps/rejected": -29.372955322265625,
      "loss": 0.848598301410675,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49112260341644287,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1017676591873169,
      "rewards/margins": 1.7578887939453125,
      "rewards/rejected": -1.6561211347579956,
      "step": 86,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.14864648406214961,
      "grad_norm": 5.431716442108154,
      "learning_rate": 0.00019968672134503213,
      "logits/chosen": -0.6225081086158752,
      "logits/rejected": -0.6252766251564026,
      "logps/chosen": -7.893375873565674,
      "logps/rejected": -24.980260848999023,
      "loss": 0.7993619441986084,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3846016228199005,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.16642117500305176,
      "rewards/margins": 1.3823862075805664,
      "rewards/rejected": -1.215964913368225,
      "step": 87,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.15035506433872603,
      "grad_norm": 3.7740750312805176,
      "learning_rate": 0.00019966395747294938,
      "logits/chosen": -0.5984699130058289,
      "logits/rejected": -0.6030604839324951,
      "logps/chosen": -5.385066032409668,
      "logps/rejected": -25.168312072753906,
      "loss": 0.6654453873634338,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3348524868488312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16900494694709778,
      "rewards/margins": 1.3018345832824707,
      "rewards/rejected": -1.1328295469284058,
      "step": 88,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.15206364461530247,
      "grad_norm": 3.6669890880584717,
      "learning_rate": 0.0001996403966905185,
      "logits/chosen": -0.6115779280662537,
      "logits/rejected": -0.615218997001648,
      "logps/chosen": -8.385238647460938,
      "logps/rejected": -20.905569076538086,
      "loss": 0.8084090948104858,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39410629868507385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07257324457168579,
      "rewards/margins": 1.0608566999435425,
      "rewards/rejected": -0.9882834553718567,
      "step": 89,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.15377222489187892,
      "grad_norm": 8.954024314880371,
      "learning_rate": 0.00019961603918613077,
      "logits/chosen": -0.6033557653427124,
      "logits/rejected": -0.6063033938407898,
      "logps/chosen": -7.85408878326416,
      "logps/rejected": -24.28641700744629,
      "loss": 0.8988556861877441,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4972013235092163,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1499406397342682,
      "rewards/margins": 1.2390244007110596,
      "rewards/rejected": -1.0890836715698242,
      "step": 90,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.15548080516845533,
      "grad_norm": 2.68121075630188,
      "learning_rate": 0.00019959088515454827,
      "logits/chosen": -0.6473177075386047,
      "logits/rejected": -0.6510601043701172,
      "logps/chosen": -6.986569404602051,
      "logps/rejected": -28.0955810546875,
      "loss": 0.6454746723175049,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3593181073665619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.310781866312027,
      "rewards/margins": 1.7279632091522217,
      "rewards/rejected": -1.417181372642517,
      "step": 91,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.15718938544503178,
      "grad_norm": 7.000541687011719,
      "learning_rate": 0.0001995649347969019,
      "logits/chosen": -0.7174542546272278,
      "logits/rejected": -0.7201290726661682,
      "logps/chosen": -11.954474449157715,
      "logps/rejected": -24.34893226623535,
      "loss": 1.0948961973190308,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6069883704185486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.005067221820354462,
      "rewards/margins": 1.2273277044296265,
      "rewards/rejected": -1.232394814491272,
      "step": 92,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1588979657216082,
      "grad_norm": 4.3811211585998535,
      "learning_rate": 0.00019953818832069,
      "logits/chosen": -0.6739280223846436,
      "logits/rejected": -0.6795160174369812,
      "logps/chosen": -6.881966590881348,
      "logps/rejected": -37.4478645324707,
      "loss": 0.6469016075134277,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40633073449134827,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.20596878230571747,
      "rewards/margins": 2.5138516426086426,
      "rewards/rejected": -2.3078830242156982,
      "step": 93,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.16060654599818464,
      "grad_norm": 12.26343822479248,
      "learning_rate": 0.00019951064593977668,
      "logits/chosen": -0.6859129667282104,
      "logits/rejected": -0.6899083852767944,
      "logps/chosen": -13.38476848602295,
      "logps/rejected": -30.595182418823242,
      "loss": 1.2918721437454224,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6223256587982178,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.30435577034950256,
      "rewards/margins": 1.605468988418579,
      "rewards/rejected": -1.9098248481750488,
      "step": 94,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.16231512627476108,
      "grad_norm": 6.570938587188721,
      "learning_rate": 0.00019948230787439017,
      "logits/chosen": -0.7028453946113586,
      "logits/rejected": -0.7050482034683228,
      "logps/chosen": -6.761815071105957,
      "logps/rejected": -27.314367294311523,
      "loss": 0.9538392424583435,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39429235458374023,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012213893234729767,
      "rewards/margins": 1.6957728862762451,
      "rewards/rejected": -1.6835590600967407,
      "step": 95,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.1640237065513375,
      "grad_norm": 2.8915793895721436,
      "learning_rate": 0.0001994531743511208,
      "logits/chosen": -0.6953604817390442,
      "logits/rejected": -0.6982015371322632,
      "logps/chosen": -7.192444801330566,
      "logps/rejected": -26.632644653320312,
      "loss": 0.6771315336227417,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3128400444984436,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.21012118458747864,
      "rewards/margins": 1.6979337930679321,
      "rewards/rejected": -1.4878127574920654,
      "step": 96,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.16573228682791394,
      "grad_norm": 2.7815051078796387,
      "learning_rate": 0.00019942324560291952,
      "logits/chosen": -0.6696122884750366,
      "logits/rejected": -0.6713677644729614,
      "logps/chosen": -4.250362396240234,
      "logps/rejected": -22.2177791595459,
      "loss": 0.6821787357330322,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26163250207901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09910638630390167,
      "rewards/margins": 1.2715519666671753,
      "rewards/rejected": -1.1724457740783691,
      "step": 97,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.16744086710449035,
      "grad_norm": 2.4311976432800293,
      "learning_rate": 0.00019939252186909574,
      "logits/chosen": -0.6763203740119934,
      "logits/rejected": -0.6752580404281616,
      "logps/chosen": -6.972185134887695,
      "logps/rejected": -21.0667667388916,
      "loss": 0.5966892838478088,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2420358657836914,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14732374250888824,
      "rewards/margins": 1.0790168046951294,
      "rewards/rejected": -0.9316931366920471,
      "step": 98,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.1691494473810668,
      "grad_norm": 5.822193622589111,
      "learning_rate": 0.00019936100339531564,
      "logits/chosen": -0.6894733905792236,
      "logits/rejected": -0.6909453272819519,
      "logps/chosen": -6.009974479675293,
      "logps/rejected": -21.0426025390625,
      "loss": 0.7569796442985535,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4093407690525055,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.26579749584198,
      "rewards/margins": 1.3290454149246216,
      "rewards/rejected": -1.0632479190826416,
      "step": 99,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.1708580276576432,
      "grad_norm": 3.2212934494018555,
      "learning_rate": 0.0001993286904336001,
      "logits/chosen": -0.7202757000923157,
      "logits/rejected": -0.7197086215019226,
      "logps/chosen": -6.131860733032227,
      "logps/rejected": -21.27735137939453,
      "loss": 0.622944712638855,
      "memory(GiB)": 46.38,
      "nll_loss": 0.28273510932922363,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26464149355888367,
      "rewards/margins": 1.1921066045761108,
      "rewards/rejected": -0.9274651408195496,
      "step": 100,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.17256660793421966,
      "grad_norm": 5.42195463180542,
      "learning_rate": 0.00019929558324232268,
      "logits/chosen": -0.7189818620681763,
      "logits/rejected": -0.720359742641449,
      "logps/chosen": -8.62668228149414,
      "logps/rejected": -19.183561325073242,
      "loss": 0.9257504940032959,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5092519521713257,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09434813261032104,
      "rewards/margins": 0.8864970803260803,
      "rewards/rejected": -0.7921488881111145,
      "step": 101,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1742751882107961,
      "grad_norm": 6.533127307891846,
      "learning_rate": 0.0001992616820862076,
      "logits/chosen": -0.6889673471450806,
      "logits/rejected": -0.6902090907096863,
      "logps/chosen": -9.093452453613281,
      "logps/rejected": -19.451143264770508,
      "loss": 0.9490825533866882,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45314493775367737,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03901132568717003,
      "rewards/margins": 0.7304368019104004,
      "rewards/rejected": -0.6914253830909729,
      "step": 102,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.17598376848737252,
      "grad_norm": 3.991544485092163,
      "learning_rate": 0.00019922698723632767,
      "logits/chosen": -0.7347920536994934,
      "logits/rejected": -0.7329245209693909,
      "logps/chosen": -10.086681365966797,
      "logps/rejected": -24.030868530273438,
      "loss": 0.8895858526229858,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5397864580154419,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.12226971983909607,
      "rewards/margins": 1.3651487827301025,
      "rewards/rejected": -1.2428791522979736,
      "step": 103,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.17769234876394896,
      "grad_norm": 4.0011162757873535,
      "learning_rate": 0.0001991914989701019,
      "logits/chosen": -0.7434298992156982,
      "logits/rejected": -0.7457666993141174,
      "logps/chosen": -8.058287620544434,
      "logps/rejected": -20.040809631347656,
      "loss": 0.7681674957275391,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3557548522949219,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.12300814688205719,
      "rewards/margins": 1.063488483428955,
      "rewards/rejected": -0.9404802918434143,
      "step": 104,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.17940092904052538,
      "grad_norm": 3.329845905303955,
      "learning_rate": 0.00019915521757129354,
      "logits/chosen": -0.7461959719657898,
      "logits/rejected": -0.7488771677017212,
      "logps/chosen": -7.418200969696045,
      "logps/rejected": -25.79469108581543,
      "loss": 0.6970200538635254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36587315797805786,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12871423363685608,
      "rewards/margins": 1.5802878141403198,
      "rewards/rejected": -1.4515737295150757,
      "step": 105,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.18110950931710182,
      "grad_norm": 4.27202033996582,
      "learning_rate": 0.00019911814333000772,
      "logits/chosen": -0.7183383703231812,
      "logits/rejected": -0.7201208472251892,
      "logps/chosen": -9.424932479858398,
      "logps/rejected": -25.1419677734375,
      "loss": 0.8681972622871399,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4449956715106964,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04691670835018158,
      "rewards/margins": 1.2326714992523193,
      "rewards/rejected": -1.1857548952102661,
      "step": 106,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.18281808959367826,
      "grad_norm": 4.578512191772461,
      "learning_rate": 0.00019908027654268903,
      "logits/chosen": -0.7675899267196655,
      "logits/rejected": -0.7682605385780334,
      "logps/chosen": -8.889389038085938,
      "logps/rejected": -24.62353515625,
      "loss": 0.8598517179489136,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40520450472831726,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13606908917427063,
      "rewards/margins": 1.3364840745925903,
      "rewards/rejected": -1.200415015220642,
      "step": 107,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.18452666987025468,
      "grad_norm": 3.067852020263672,
      "learning_rate": 0.00019904161751211938,
      "logits/chosen": -0.7350285649299622,
      "logits/rejected": -0.7360118627548218,
      "logps/chosen": -12.491768836975098,
      "logps/rejected": -23.78077507019043,
      "loss": 0.8337110280990601,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4792723059654236,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0716579332947731,
      "rewards/margins": 1.2602936029434204,
      "rewards/rejected": -1.1886355876922607,
      "step": 108,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.18623525014683112,
      "grad_norm": 3.2055752277374268,
      "learning_rate": 0.00019900216654741528,
      "logits/chosen": -0.7312414646148682,
      "logits/rejected": -0.7316892147064209,
      "logps/chosen": -8.460771560668945,
      "logps/rejected": -21.30421257019043,
      "loss": 0.7292563319206238,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3923528492450714,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2296271026134491,
      "rewards/margins": 1.3531551361083984,
      "rewards/rejected": -1.1235278844833374,
      "step": 109,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.18794383042340754,
      "grad_norm": 4.93084716796875,
      "learning_rate": 0.0001989619239640256,
      "logits/chosen": -0.7337685823440552,
      "logits/rejected": -0.7344739437103271,
      "logps/chosen": -11.766962051391602,
      "logps/rejected": -22.844139099121094,
      "loss": 0.8851971626281738,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4795154333114624,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06328105181455612,
      "rewards/margins": 1.133680820465088,
      "rewards/rejected": -1.0703996419906616,
      "step": 110,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.18965241069998398,
      "grad_norm": 10.518948554992676,
      "learning_rate": 0.00019892089008372897,
      "logits/chosen": -0.709618330001831,
      "logits/rejected": -0.7111095190048218,
      "logps/chosen": -8.326457977294922,
      "logps/rejected": -27.638351440429688,
      "loss": 0.9548771977424622,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5345268249511719,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.045853935182094574,
      "rewards/margins": 1.551784634590149,
      "rewards/rejected": -1.5059306621551514,
      "step": 111,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.19136099097656042,
      "grad_norm": 7.465288162231445,
      "learning_rate": 0.0001988790652346312,
      "logits/chosen": -0.7286843657493591,
      "logits/rejected": -0.729621946811676,
      "logps/chosen": -8.483978271484375,
      "logps/rejected": -28.15561294555664,
      "loss": 0.922063410282135,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5344014167785645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13854525983333588,
      "rewards/margins": 1.4421772956848145,
      "rewards/rejected": -1.3036320209503174,
      "step": 112,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.19306957125313684,
      "grad_norm": 8.20088005065918,
      "learning_rate": 0.0001988364497511627,
      "logits/chosen": -0.740546703338623,
      "logits/rejected": -0.7427754402160645,
      "logps/chosen": -8.096268653869629,
      "logps/rejected": -22.899229049682617,
      "loss": 0.9872632622718811,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5750774145126343,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18856163322925568,
      "rewards/margins": 1.2329988479614258,
      "rewards/rejected": -1.0444371700286865,
      "step": 113,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.19477815152971328,
      "grad_norm": 3.4554946422576904,
      "learning_rate": 0.0001987930439740757,
      "logits/chosen": -0.699242115020752,
      "logits/rejected": -0.7016286849975586,
      "logps/chosen": -11.758835792541504,
      "logps/rejected": -22.287220001220703,
      "loss": 0.7449758052825928,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4075641632080078,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1979522556066513,
      "rewards/margins": 1.2950595617294312,
      "rewards/rejected": -1.097107172012329,
      "step": 114,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.1964867318062897,
      "grad_norm": 2.8801109790802,
      "learning_rate": 0.00019874884825044165,
      "logits/chosen": -0.673065185546875,
      "logits/rejected": -0.6760713458061218,
      "logps/chosen": -6.409548759460449,
      "logps/rejected": -16.89990234375,
      "loss": 0.7451252937316895,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31410443782806396,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.19196031987667084,
      "rewards/margins": 0.9307445287704468,
      "rewards/rejected": -0.7387841939926147,
      "step": 115,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.19819531208286614,
      "grad_norm": 2.597715377807617,
      "learning_rate": 0.00019870386293364836,
      "logits/chosen": -0.642209529876709,
      "logits/rejected": -0.6442127227783203,
      "logps/chosen": -6.406985282897949,
      "logps/rejected": -19.21172523498535,
      "loss": 0.7881473898887634,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3131154477596283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14077159762382507,
      "rewards/margins": 0.9926955699920654,
      "rewards/rejected": -0.851923942565918,
      "step": 116,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.1999038923594426,
      "grad_norm": 2.9220879077911377,
      "learning_rate": 0.00019865808838339727,
      "logits/chosen": -0.6920750141143799,
      "logits/rejected": -0.6941624283790588,
      "logps/chosen": -7.081650733947754,
      "logps/rejected": -24.63327407836914,
      "loss": 0.6955921649932861,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29941731691360474,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.014344778843224049,
      "rewards/margins": 1.21220862865448,
      "rewards/rejected": -1.1978638172149658,
      "step": 117,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.201612472636019,
      "grad_norm": 2.535001277923584,
      "learning_rate": 0.00019861152496570043,
      "logits/chosen": -0.6474766135215759,
      "logits/rejected": -0.6474269032478333,
      "logps/chosen": -4.845932483673096,
      "logps/rejected": -30.43771743774414,
      "loss": 0.549997866153717,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2411210685968399,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22833456099033356,
      "rewards/margins": 1.7972755432128906,
      "rewards/rejected": -1.5689408779144287,
      "step": 118,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.20332105291259545,
      "grad_norm": 2.592416286468506,
      "learning_rate": 0.00019856417305287772,
      "logits/chosen": -0.6813372373580933,
      "logits/rejected": -0.684921383857727,
      "logps/chosen": -8.64324951171875,
      "logps/rejected": -25.918020248413086,
      "loss": 0.6950156092643738,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34918904304504395,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.27043017745018005,
      "rewards/margins": 1.5927720069885254,
      "rewards/rejected": -1.322341799736023,
      "step": 119,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.20502963318917186,
      "grad_norm": 3.0879509449005127,
      "learning_rate": 0.00019851603302355373,
      "logits/chosen": -0.7103255987167358,
      "logits/rejected": -0.7141609191894531,
      "logps/chosen": -7.374659061431885,
      "logps/rejected": -27.09424591064453,
      "loss": 0.7110822200775146,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3552224040031433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15158146619796753,
      "rewards/margins": 1.6895453929901123,
      "rewards/rejected": -1.5379639863967896,
      "step": 120,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.2067382134657483,
      "grad_norm": 3.703174591064453,
      "learning_rate": 0.00019846710526265487,
      "logits/chosen": -0.7088285088539124,
      "logits/rejected": -0.7127594947814941,
      "logps/chosen": -6.05606746673584,
      "logps/rejected": -27.269733428955078,
      "loss": 0.619713306427002,
      "memory(GiB)": 46.38,
      "nll_loss": 0.333889901638031,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2516608238220215,
      "rewards/margins": 1.8369193077087402,
      "rewards/rejected": -1.5852584838867188,
      "step": 121,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.20844679374232475,
      "grad_norm": 4.171956539154053,
      "learning_rate": 0.00019841739016140624,
      "logits/chosen": -0.7836604118347168,
      "logits/rejected": -0.7841136455535889,
      "logps/chosen": -7.950913429260254,
      "logps/rejected": -26.353294372558594,
      "loss": 0.7199419736862183,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3016027510166168,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03286279737949371,
      "rewards/margins": 1.4103941917419434,
      "rewards/rejected": -1.3775315284729004,
      "step": 122,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.21015537401890116,
      "grad_norm": 5.59217643737793,
      "learning_rate": 0.0001983668881173284,
      "logits/chosen": -0.7123353481292725,
      "logits/rejected": -0.7176805734634399,
      "logps/chosen": -6.9763007164001465,
      "logps/rejected": -40.705894470214844,
      "loss": 0.5055665969848633,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2759905159473419,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13582277297973633,
      "rewards/margins": 2.353527069091797,
      "rewards/rejected": -2.2177045345306396,
      "step": 123,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.2118639542954776,
      "grad_norm": 7.052844524383545,
      "learning_rate": 0.00019831559953423442,
      "logits/chosen": -0.7205808758735657,
      "logits/rejected": -0.7224647402763367,
      "logps/chosen": -8.208627700805664,
      "logps/rejected": -23.56304359436035,
      "loss": 0.8278493285179138,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3575640022754669,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0022782105952501297,
      "rewards/margins": 1.129638910293579,
      "rewards/rejected": -1.1273607015609741,
      "step": 124,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.21357253457205402,
      "grad_norm": 2.853487491607666,
      "learning_rate": 0.00019826352482222638,
      "logits/chosen": -0.7876097559928894,
      "logits/rejected": -0.7900496125221252,
      "logps/chosen": -6.370059013366699,
      "logps/rejected": -29.205093383789062,
      "loss": 0.553210437297821,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24043366312980652,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.03330150991678238,
      "rewards/margins": 1.9821114540100098,
      "rewards/rejected": -1.9488096237182617,
      "step": 125,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.21528111484863047,
      "grad_norm": 3.3915181159973145,
      "learning_rate": 0.00019821066439769234,
      "logits/chosen": -0.7244046926498413,
      "logits/rejected": -0.7288062572479248,
      "logps/chosen": -7.876413822174072,
      "logps/rejected": -35.36836242675781,
      "loss": 0.5778641104698181,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34048542380332947,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.027921676635742188,
      "rewards/margins": 2.1973161697387695,
      "rewards/rejected": -2.225238084793091,
      "step": 126,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.2169896951252069,
      "grad_norm": 2.667902946472168,
      "learning_rate": 0.00019815701868330284,
      "logits/chosen": -0.7135522365570068,
      "logits/rejected": -0.7176554799079895,
      "logps/chosen": -11.242607116699219,
      "logps/rejected": -33.319515228271484,
      "loss": 0.7801873087882996,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49927234649658203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23649244010448456,
      "rewards/margins": 2.445638656616211,
      "rewards/rejected": -2.209146022796631,
      "step": 127,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.21869827540178333,
      "grad_norm": 9.078632354736328,
      "learning_rate": 0.00019810258810800752,
      "logits/chosen": -0.7386202812194824,
      "logits/rejected": -0.7396747469902039,
      "logps/chosen": -9.633708000183105,
      "logps/rejected": -30.443008422851562,
      "loss": 0.7859803438186646,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49833089113235474,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04507865756750107,
      "rewards/margins": 1.9784715175628662,
      "rewards/rejected": -1.933392882347107,
      "step": 128,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.22040685567835977,
      "grad_norm": 13.282059669494629,
      "learning_rate": 0.0001980473731070319,
      "logits/chosen": -0.7503476142883301,
      "logits/rejected": -0.755096971988678,
      "logps/chosen": -7.456392765045166,
      "logps/rejected": -29.754714965820312,
      "loss": 0.8027007579803467,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4415196180343628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14411740005016327,
      "rewards/margins": 2.093838691711426,
      "rewards/rejected": -1.949721336364746,
      "step": 129,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.22211543595493619,
      "grad_norm": 5.136019706726074,
      "learning_rate": 0.0001979913741218736,
      "logits/chosen": -0.7441588640213013,
      "logits/rejected": -0.7470377683639526,
      "logps/chosen": -11.379145622253418,
      "logps/rejected": -36.20026397705078,
      "loss": 0.9255682229995728,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5846636891365051,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.014721579849720001,
      "rewards/margins": 2.245670795440674,
      "rewards/rejected": -2.2309491634368896,
      "step": 130,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.22382401623151263,
      "grad_norm": 5.494217395782471,
      "learning_rate": 0.00019793459160029903,
      "logits/chosen": -0.7520757913589478,
      "logits/rejected": -0.7574396133422852,
      "logps/chosen": -6.406137466430664,
      "logps/rejected": -42.744041442871094,
      "loss": 0.6142405867576599,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3836834132671356,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.19515469670295715,
      "rewards/margins": 3.062283992767334,
      "rewards/rejected": -2.867129325866699,
      "step": 131,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.22553259650808907,
      "grad_norm": 4.463568687438965,
      "learning_rate": 0.00019787702599633971,
      "logits/chosen": -0.7327495217323303,
      "logits/rejected": -0.738735556602478,
      "logps/chosen": -6.503672122955322,
      "logps/rejected": -39.14170837402344,
      "loss": 0.7878039479255676,
      "memory(GiB)": 46.38,
      "nll_loss": 0.37348490953445435,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.034972380846738815,
      "rewards/margins": 2.3170716762542725,
      "rewards/rejected": -2.352044105529785,
      "step": 132,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.2272411767846655,
      "grad_norm": 7.909404754638672,
      "learning_rate": 0.0001978186777702887,
      "logits/chosen": -0.6934617757797241,
      "logits/rejected": -0.6948121786117554,
      "logps/chosen": -7.257785797119141,
      "logps/rejected": -27.908708572387695,
      "loss": 0.7475972175598145,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3585331439971924,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.036370277404785156,
      "rewards/margins": 1.666322946548462,
      "rewards/rejected": -1.6299527883529663,
      "step": 133,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.22894975706124193,
      "grad_norm": 14.575544357299805,
      "learning_rate": 0.00019775954738869683,
      "logits/chosen": -0.6997110843658447,
      "logits/rejected": -0.7026513814926147,
      "logps/chosen": -7.946556091308594,
      "logps/rejected": -20.762996673583984,
      "loss": 1.056870460510254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6803016066551208,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16680209338665009,
      "rewards/margins": 1.3951897621154785,
      "rewards/rejected": -1.2283875942230225,
      "step": 134,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.23065833733781835,
      "grad_norm": 6.814887046813965,
      "learning_rate": 0.00019769963532436911,
      "logits/chosen": -0.661209762096405,
      "logits/rejected": -0.6659980416297913,
      "logps/chosen": -8.108284950256348,
      "logps/rejected": -26.44216537475586,
      "loss": 1.0938985347747803,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7105523347854614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.010204421356320381,
      "rewards/margins": 1.3176696300506592,
      "rewards/rejected": -1.3074650764465332,
      "step": 135,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.2323669176143948,
      "grad_norm": 5.490352153778076,
      "learning_rate": 0.00019763894205636072,
      "logits/chosen": -0.671430230140686,
      "logits/rejected": -0.673378586769104,
      "logps/chosen": -9.945783615112305,
      "logps/rejected": -22.1285343170166,
      "loss": 0.976132333278656,
      "memory(GiB)": 46.38,
      "nll_loss": 0.543598473072052,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1351984441280365,
      "rewards/margins": 1.1337149143218994,
      "rewards/rejected": -0.9985164403915405,
      "step": 136,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.23407549789097123,
      "grad_norm": 5.260707378387451,
      "learning_rate": 0.00019757746806997348,
      "logits/chosen": -0.6116827130317688,
      "logits/rejected": -0.6123840808868408,
      "logps/chosen": -8.495226860046387,
      "logps/rejected": -17.80582046508789,
      "loss": 0.9422957897186279,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3756696879863739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009481308050453663,
      "rewards/margins": 0.6523244380950928,
      "rewards/rejected": -0.6618058085441589,
      "step": 137,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.23578407816754765,
      "grad_norm": 2.1559102535247803,
      "learning_rate": 0.00019751521385675172,
      "logits/chosen": -0.6613444089889526,
      "logits/rejected": -0.6647210121154785,
      "logps/chosen": -4.250008583068848,
      "logps/rejected": -25.66912078857422,
      "loss": 0.5518596172332764,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1721974015235901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17000286281108856,
      "rewards/margins": 1.491791009902954,
      "rewards/rejected": -1.3217883110046387,
      "step": 138,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.2374926584441241,
      "grad_norm": 2.9551846981048584,
      "learning_rate": 0.00019745217991447848,
      "logits/chosen": -0.6719025373458862,
      "logits/rejected": -0.6741494536399841,
      "logps/chosen": -8.146540641784668,
      "logps/rejected": -28.308319091796875,
      "loss": 0.7604531049728394,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34055855870246887,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1295825093984604,
      "rewards/margins": 1.5078970193862915,
      "rewards/rejected": -1.37831449508667,
      "step": 139,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.2392012387207005,
      "grad_norm": 5.525161266326904,
      "learning_rate": 0.0001973883667471715,
      "logits/chosen": -0.6694546937942505,
      "logits/rejected": -0.6737231612205505,
      "logps/chosen": -10.553884506225586,
      "logps/rejected": -32.512393951416016,
      "loss": 0.822769284248352,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3502211570739746,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01343974657356739,
      "rewards/margins": 1.7657444477081299,
      "rewards/rejected": -1.7523046731948853,
      "step": 140,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.24090981899727695,
      "grad_norm": 2.4956538677215576,
      "learning_rate": 0.00019732377486507915,
      "logits/chosen": -0.6616899371147156,
      "logits/rejected": -0.6654747724533081,
      "logps/chosen": -6.116077423095703,
      "logps/rejected": -22.98920440673828,
      "loss": 0.7530378103256226,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33646857738494873,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09931784868240356,
      "rewards/margins": 1.2228065729141235,
      "rewards/rejected": -1.1234887838363647,
      "step": 141,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.24261839927385337,
      "grad_norm": 3.0544512271881104,
      "learning_rate": 0.00019725840478467636,
      "logits/chosen": -0.6879534125328064,
      "logits/rejected": -0.6862825155258179,
      "logps/chosen": -8.299098014831543,
      "logps/rejected": -18.718904495239258,
      "loss": 0.7917457818984985,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31696975231170654,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.11313430219888687,
      "rewards/margins": 0.9731096029281616,
      "rewards/rejected": -0.859975278377533,
      "step": 142,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.2443269795504298,
      "grad_norm": 2.7635226249694824,
      "learning_rate": 0.00019719225702866062,
      "logits/chosen": -0.7383114099502563,
      "logits/rejected": -0.736755907535553,
      "logps/chosen": -5.972444534301758,
      "logps/rejected": -27.483884811401367,
      "loss": 0.6677893400192261,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3221934735774994,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15736974775791168,
      "rewards/margins": 1.7971253395080566,
      "rewards/rejected": -1.639755368232727,
      "step": 143,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.24603555982700626,
      "grad_norm": 5.206491470336914,
      "learning_rate": 0.0001971253321259476,
      "logits/chosen": -0.7135074138641357,
      "logits/rejected": -0.7155020833015442,
      "logps/chosen": -9.462960243225098,
      "logps/rejected": -22.44765853881836,
      "loss": 0.8617446422576904,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4634811282157898,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1327909231185913,
      "rewards/margins": 1.3640327453613281,
      "rewards/rejected": -1.2312417030334473,
      "step": 144,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.24774414010358267,
      "grad_norm": 2.5136232376098633,
      "learning_rate": 0.00019705763061166696,
      "logits/chosen": -0.7309613227844238,
      "logits/rejected": -0.7339869141578674,
      "logps/chosen": -8.707137107849121,
      "logps/rejected": -30.160343170166016,
      "loss": 0.6045644879341125,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31675195693969727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1917949914932251,
      "rewards/margins": 2.2541422843933105,
      "rewards/rejected": -2.062347412109375,
      "step": 145,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.24945272038015912,
      "grad_norm": 3.6773223876953125,
      "learning_rate": 0.00019698915302715836,
      "logits/chosen": -0.7318214774131775,
      "logits/rejected": -0.7343602180480957,
      "logps/chosen": -6.429262161254883,
      "logps/rejected": -31.925209045410156,
      "loss": 0.6257779598236084,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31483885645866394,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1830574870109558,
      "rewards/margins": 1.988149642944336,
      "rewards/rejected": -1.8050918579101562,
      "step": 146,
      "train_speed(iter/s)": 0.005505
    },
    {
      "epoch": 0.25116130065673553,
      "grad_norm": 9.478280067443848,
      "learning_rate": 0.00019691989991996663,
      "logits/chosen": -0.7461144924163818,
      "logits/rejected": -0.7457807660102844,
      "logps/chosen": -10.189486503601074,
      "logps/rejected": -34.494956970214844,
      "loss": 0.7869279980659485,
      "memory(GiB)": 46.38,
      "nll_loss": 0.47246837615966797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1757950335741043,
      "rewards/margins": 2.238186836242676,
      "rewards/rejected": -2.062391996383667,
      "step": 147,
      "train_speed(iter/s)": 0.005504
    },
    {
      "epoch": 0.252869880933312,
      "grad_norm": 7.141703128814697,
      "learning_rate": 0.00019684987184383776,
      "logits/chosen": -0.7700251340866089,
      "logits/rejected": -0.7725012302398682,
      "logps/chosen": -6.46027946472168,
      "logps/rejected": -34.821231842041016,
      "loss": 0.692828357219696,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3844330608844757,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12604108452796936,
      "rewards/margins": 2.6629996299743652,
      "rewards/rejected": -2.5369582176208496,
      "step": 148,
      "train_speed(iter/s)": 0.005505
    },
    {
      "epoch": 0.2545784612098884,
      "grad_norm": 14.823970794677734,
      "learning_rate": 0.00019677906935871447,
      "logits/chosen": -0.7374511361122131,
      "logits/rejected": -0.7389614582061768,
      "logps/chosen": -10.582510948181152,
      "logps/rejected": -26.03261375427246,
      "loss": 1.33296537399292,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6253257989883423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18944962322711945,
      "rewards/margins": 1.5741108655929565,
      "rewards/rejected": -1.7635605335235596,
      "step": 149,
      "train_speed(iter/s)": 0.005504
    },
    {
      "epoch": 0.25628704148646486,
      "grad_norm": 13.657255172729492,
      "learning_rate": 0.00019670749303073146,
      "logits/chosen": -0.7172877192497253,
      "logits/rejected": -0.7211941480636597,
      "logps/chosen": -6.75715970993042,
      "logps/rejected": -41.68476867675781,
      "loss": 0.7531163692474365,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4120977818965912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10011129081249237,
      "rewards/margins": 3.0644404888153076,
      "rewards/rejected": -2.964329481124878,
      "step": 150,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.25799562176304125,
      "grad_norm": 4.298210144042969,
      "learning_rate": 0.00019663514343221117,
      "logits/chosen": -0.6899698972702026,
      "logits/rejected": -0.6891142725944519,
      "logps/chosen": -9.789582252502441,
      "logps/rejected": -32.59271240234375,
      "loss": 0.6402732729911804,
      "memory(GiB)": 46.38,
      "nll_loss": 0.335668683052063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15986008942127228,
      "rewards/margins": 2.269547939300537,
      "rewards/rejected": -2.1096878051757812,
      "step": 151,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.2597042020396177,
      "grad_norm": 4.9224324226379395,
      "learning_rate": 0.00019656202114165904,
      "logits/chosen": -0.717603862285614,
      "logits/rejected": -0.7218050956726074,
      "logps/chosen": -8.468698501586914,
      "logps/rejected": -42.157318115234375,
      "loss": 0.6935219168663025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4874541759490967,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.11759095638990402,
      "rewards/margins": 2.7493767738342285,
      "rewards/rejected": -2.8669681549072266,
      "step": 152,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.26141278231619414,
      "grad_norm": 5.180264472961426,
      "learning_rate": 0.00019648812674375902,
      "logits/chosen": -0.6897304058074951,
      "logits/rejected": -0.6921338438987732,
      "logps/chosen": -6.188149929046631,
      "logps/rejected": -35.15382385253906,
      "loss": 0.6033867597579956,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2803235948085785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18143904209136963,
      "rewards/margins": 2.3970108032226562,
      "rewards/rejected": -2.215571641921997,
      "step": 153,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.2631213625927706,
      "grad_norm": 4.898785591125488,
      "learning_rate": 0.00019641346082936872,
      "logits/chosen": -0.6851431131362915,
      "logits/rejected": -0.6863937973976135,
      "logps/chosen": -14.464362144470215,
      "logps/rejected": -23.472118377685547,
      "loss": 0.783013105392456,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4414927363395691,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01941562071442604,
      "rewards/margins": 1.5958939790725708,
      "rewards/rejected": -1.576478362083435,
      "step": 154,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.264829942869347,
      "grad_norm": 5.930595874786377,
      "learning_rate": 0.00019633802399551473,
      "logits/chosen": -0.7089685797691345,
      "logits/rejected": -0.7100527882575989,
      "logps/chosen": -9.57446575164795,
      "logps/rejected": -26.301237106323242,
      "loss": 0.8949770927429199,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4963584840297699,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03665952757000923,
      "rewards/margins": 1.5139617919921875,
      "rewards/rejected": -1.5506211519241333,
      "step": 155,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.2665385231459234,
      "grad_norm": 4.985401630401611,
      "learning_rate": 0.00019626181684538808,
      "logits/chosen": -0.6955317854881287,
      "logits/rejected": -0.7012707591056824,
      "logps/chosen": -6.044739723205566,
      "logps/rejected": -24.839508056640625,
      "loss": 0.9819079637527466,
      "memory(GiB)": 46.38,
      "nll_loss": 0.47508248686790466,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.06785830110311508,
      "rewards/margins": 0.7281482219696045,
      "rewards/rejected": -0.660290002822876,
      "step": 156,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.26824710342249986,
      "grad_norm": 6.6396403312683105,
      "learning_rate": 0.000196184839988339,
      "logits/chosen": -0.6605405211448669,
      "logits/rejected": -0.6651955842971802,
      "logps/chosen": -5.8500542640686035,
      "logps/rejected": -27.22307586669922,
      "loss": 0.7894259095191956,
      "memory(GiB)": 46.38,
      "nll_loss": 0.44944238662719727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20848321914672852,
      "rewards/margins": 1.5052435398101807,
      "rewards/rejected": -1.2967603206634521,
      "step": 157,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2699556836990763,
      "grad_norm": 6.331238269805908,
      "learning_rate": 0.00019610709403987246,
      "logits/chosen": -0.6915969848632812,
      "logits/rejected": -0.6922619938850403,
      "logps/chosen": -6.449377059936523,
      "logps/rejected": -22.124189376831055,
      "loss": 0.7714009881019592,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4102553725242615,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15198560059070587,
      "rewards/margins": 1.139552116394043,
      "rewards/rejected": -0.9875664710998535,
      "step": 158,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.27166426397565274,
      "grad_norm": 4.062625408172607,
      "learning_rate": 0.00019602857962164297,
      "logits/chosen": -0.7070209383964539,
      "logits/rejected": -0.7094908952713013,
      "logps/chosen": -8.760738372802734,
      "logps/rejected": -21.83628273010254,
      "loss": 0.6808652877807617,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30313965678215027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16821521520614624,
      "rewards/margins": 1.2700836658477783,
      "rewards/rejected": -1.1018683910369873,
      "step": 159,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.2733728442522292,
      "grad_norm": 3.5334150791168213,
      "learning_rate": 0.00019594929736144976,
      "logits/chosen": -0.6827138066291809,
      "logits/rejected": -0.6853998303413391,
      "logps/chosen": -8.673670768737793,
      "logps/rejected": -22.576623916625977,
      "loss": 0.8358533978462219,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4210669696331024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1305871605873108,
      "rewards/margins": 1.2085453271865845,
      "rewards/rejected": -1.077958106994629,
      "step": 160,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2750814245288056,
      "grad_norm": 4.968493938446045,
      "learning_rate": 0.0001958692478932316,
      "logits/chosen": -0.6895708441734314,
      "logits/rejected": -0.6913452744483948,
      "logps/chosen": -8.595847129821777,
      "logps/rejected": -15.964265823364258,
      "loss": 0.9963687658309937,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4303150177001953,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.039102740585803986,
      "rewards/margins": 0.7386963367462158,
      "rewards/rejected": -0.6995935440063477,
      "step": 161,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.276790004805382,
      "grad_norm": 3.1100776195526123,
      "learning_rate": 0.00019578843185706198,
      "logits/chosen": -0.6581685543060303,
      "logits/rejected": -0.6606261730194092,
      "logps/chosen": -8.823217391967773,
      "logps/rejected": -20.12518310546875,
      "loss": 0.7960190773010254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3306158185005188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1894218921661377,
      "rewards/margins": 0.9922627210617065,
      "rewards/rejected": -0.8028407096862793,
      "step": 162,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.27849858508195846,
      "grad_norm": 2.9061076641082764,
      "learning_rate": 0.00019570684989914368,
      "logits/chosen": -0.7005666494369507,
      "logits/rejected": -0.7033515572547913,
      "logps/chosen": -6.614690780639648,
      "logps/rejected": -23.320758819580078,
      "loss": 0.7204984426498413,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3370620608329773,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.20666998624801636,
      "rewards/margins": 1.309868335723877,
      "rewards/rejected": -1.1031984090805054,
      "step": 163,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.2802071653585349,
      "grad_norm": 2.7546348571777344,
      "learning_rate": 0.00019562450267180396,
      "logits/chosen": -0.713110089302063,
      "logits/rejected": -0.7162569165229797,
      "logps/chosen": -11.556283950805664,
      "logps/rejected": -24.4801025390625,
      "loss": 0.600546658039093,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29867106676101685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32357320189476013,
      "rewards/margins": 1.2928061485290527,
      "rewards/rejected": -0.969232976436615,
      "step": 164,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.28191574563511135,
      "grad_norm": 4.433908462524414,
      "learning_rate": 0.000195541390833489,
      "logits/chosen": -0.7190208435058594,
      "logits/rejected": -0.7217738628387451,
      "logps/chosen": -8.067594528198242,
      "logps/rejected": -28.02341651916504,
      "loss": 0.6388089656829834,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3460094630718231,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26204991340637207,
      "rewards/margins": 1.4149954319000244,
      "rewards/rejected": -1.152945637702942,
      "step": 165,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.28362432591168774,
      "grad_norm": 2.649824380874634,
      "learning_rate": 0.0001954575150487589,
      "logits/chosen": -0.749485969543457,
      "logits/rejected": -0.7498363852500916,
      "logps/chosen": -8.293320655822754,
      "logps/rejected": -18.8927001953125,
      "loss": 0.6769586205482483,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2774519920349121,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.14119945466518402,
      "rewards/margins": 1.063216209411621,
      "rewards/rejected": -0.9220167398452759,
      "step": 166,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2853329061882642,
      "grad_norm": 5.596877574920654,
      "learning_rate": 0.0001953728759882821,
      "logits/chosen": -0.7324296832084656,
      "logits/rejected": -0.7348483204841614,
      "logps/chosen": -10.596734046936035,
      "logps/rejected": -26.180767059326172,
      "loss": 0.6137089729309082,
      "memory(GiB)": 46.38,
      "nll_loss": 0.338160902261734,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13076083362102509,
      "rewards/margins": 1.6062695980072021,
      "rewards/rejected": -1.4755089282989502,
      "step": 167,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.2870414864648406,
      "grad_norm": 3.771359443664551,
      "learning_rate": 0.00019528747432883035,
      "logits/chosen": -0.790438711643219,
      "logits/rejected": -0.794373631477356,
      "logps/chosen": -10.444026947021484,
      "logps/rejected": -26.96920394897461,
      "loss": 0.9520989060401917,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5127608776092529,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06649212539196014,
      "rewards/margins": 1.7065045833587646,
      "rewards/rejected": -1.6400126218795776,
      "step": 168,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.28875006674141707,
      "grad_norm": 5.279207706451416,
      "learning_rate": 0.00019520131075327298,
      "logits/chosen": -0.811764657497406,
      "logits/rejected": -0.8173067569732666,
      "logps/chosen": -5.237765312194824,
      "logps/rejected": -34.11195755004883,
      "loss": 0.5845170617103577,
      "memory(GiB)": 46.38,
      "nll_loss": 0.320607453584671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16728658974170685,
      "rewards/margins": 2.271721363067627,
      "rewards/rejected": -2.1044344902038574,
      "step": 169,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.2904586470179935,
      "grad_norm": 3.968066453933716,
      "learning_rate": 0.00019511438595057174,
      "logits/chosen": -0.8485198020935059,
      "logits/rejected": -0.8553695678710938,
      "logps/chosen": -2.880854606628418,
      "logps/rejected": -36.118629455566406,
      "loss": 0.4797823131084442,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23597285151481628,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.19838106632232666,
      "rewards/margins": 2.7422218322753906,
      "rewards/rejected": -2.5438406467437744,
      "step": 170,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2921672272945699,
      "grad_norm": 5.908665180206299,
      "learning_rate": 0.00019502670061577498,
      "logits/chosen": -0.8754770159721375,
      "logits/rejected": -0.8793540000915527,
      "logps/chosen": -6.855566501617432,
      "logps/rejected": -41.512691497802734,
      "loss": 0.5422110557556152,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29031190276145935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1825065314769745,
      "rewards/margins": 3.2673420906066895,
      "rewards/rejected": -3.0848352909088135,
      "step": 171,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.29387580757114634,
      "grad_norm": 9.865046501159668,
      "learning_rate": 0.00019493825545001227,
      "logits/chosen": -0.8389215469360352,
      "logits/rejected": -0.8400964736938477,
      "logps/chosen": -5.75340461730957,
      "logps/rejected": -28.154647827148438,
      "loss": 0.5484858751296997,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24028873443603516,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0924752876162529,
      "rewards/margins": 2.016855001449585,
      "rewards/rejected": -1.9243797063827515,
      "step": 172,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2955843878477228,
      "grad_norm": 5.110999584197998,
      "learning_rate": 0.00019484905116048883,
      "logits/chosen": -0.8424903750419617,
      "logits/rejected": -0.8479667901992798,
      "logps/chosen": -10.439705848693848,
      "logps/rejected": -34.95732116699219,
      "loss": 0.9751264452934265,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6046102046966553,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.047889530658721924,
      "rewards/margins": 2.1707029342651367,
      "rewards/rejected": -2.1228134632110596,
      "step": 173,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.29729296812429923,
      "grad_norm": 5.233139514923096,
      "learning_rate": 0.00019475908846047982,
      "logits/chosen": -0.8389162421226501,
      "logits/rejected": -0.8423767685890198,
      "logps/chosen": -6.015560626983643,
      "logps/rejected": -33.967063903808594,
      "loss": 0.5880640745162964,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36994248628616333,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.29298272728919983,
      "rewards/margins": 2.3524224758148193,
      "rewards/rejected": -2.0594396591186523,
      "step": 174,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.29900154840087567,
      "grad_norm": 3.566279649734497,
      "learning_rate": 0.00019466836806932453,
      "logits/chosen": -0.8825660943984985,
      "logits/rejected": -0.8887041807174683,
      "logps/chosen": -6.019082546234131,
      "logps/rejected": -35.091529846191406,
      "loss": 0.6474583148956299,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41612136363983154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2304912507534027,
      "rewards/margins": 2.494393825531006,
      "rewards/rejected": -2.2639026641845703,
      "step": 175,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.30071012867745206,
      "grad_norm": 6.7856950759887695,
      "learning_rate": 0.00019457689071242085,
      "logits/chosen": -0.8579093813896179,
      "logits/rejected": -0.8634753227233887,
      "logps/chosen": -9.888270378112793,
      "logps/rejected": -41.228416442871094,
      "loss": 0.6879767179489136,
      "memory(GiB)": 46.38,
      "nll_loss": 0.423296183347702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31314119696617126,
      "rewards/margins": 2.737194538116455,
      "rewards/rejected": -2.424053192138672,
      "step": 176,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3024187089540285,
      "grad_norm": 4.894569396972656,
      "learning_rate": 0.0001944846571212192,
      "logits/chosen": -0.8246264457702637,
      "logits/rejected": -0.8278872966766357,
      "logps/chosen": -9.956787109375,
      "logps/rejected": -38.914066314697266,
      "loss": 0.7229227423667908,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3963947594165802,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.03048780933022499,
      "rewards/margins": 2.575648307800293,
      "rewards/rejected": -2.5451607704162598,
      "step": 177,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.30412728923060495,
      "grad_norm": 5.153270721435547,
      "learning_rate": 0.000194391668033217,
      "logits/chosen": -0.8741589784622192,
      "logits/rejected": -0.879849374294281,
      "logps/chosen": -5.598626613616943,
      "logps/rejected": -33.6767463684082,
      "loss": 0.6376072764396667,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31503573060035706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20653334259986877,
      "rewards/margins": 2.248800754547119,
      "rewards/rejected": -2.0422675609588623,
      "step": 178,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3058358695071814,
      "grad_norm": 6.50536584854126,
      "learning_rate": 0.00019429792419195247,
      "logits/chosen": -0.83283931016922,
      "logits/rejected": -0.8357385993003845,
      "logps/chosen": -5.801810264587402,
      "logps/rejected": -26.60141372680664,
      "loss": 0.7859883308410645,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40051573514938354,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.022562842816114426,
      "rewards/margins": 1.451593041419983,
      "rewards/rejected": -1.429030179977417,
      "step": 179,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.30754444978375783,
      "grad_norm": 2.6014249324798584,
      "learning_rate": 0.0001942034263469989,
      "logits/chosen": -0.8176060914993286,
      "logits/rejected": -0.8231288194656372,
      "logps/chosen": -8.629841804504395,
      "logps/rejected": -33.58430480957031,
      "loss": 0.6443658471107483,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34101858735084534,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10939686745405197,
      "rewards/margins": 2.1205523014068604,
      "rewards/rejected": -2.011155605316162,
      "step": 180,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3092530300603342,
      "grad_norm": 1.8803960084915161,
      "learning_rate": 0.00019410817525395855,
      "logits/chosen": -0.8076679706573486,
      "logits/rejected": -0.8171229958534241,
      "logps/chosen": -3.606616735458374,
      "logps/rejected": -41.894737243652344,
      "loss": 0.37792402505874634,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1496608406305313,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.18168124556541443,
      "rewards/margins": 2.5405263900756836,
      "rewards/rejected": -2.3588449954986572,
      "step": 181,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.31096161033691067,
      "grad_norm": 1.7322460412979126,
      "learning_rate": 0.0001940121716744566,
      "logits/chosen": -0.8273515701293945,
      "logits/rejected": -0.8358028531074524,
      "logps/chosen": -4.195955753326416,
      "logps/rejected": -42.863243103027344,
      "loss": 0.40653136372566223,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1898367702960968,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2309657633304596,
      "rewards/margins": 2.9126315116882324,
      "rewards/rejected": -2.6816658973693848,
      "step": 182,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.3126701906134871,
      "grad_norm": 7.253870010375977,
      "learning_rate": 0.00019391541637613508,
      "logits/chosen": -0.818758487701416,
      "logits/rejected": -0.8240275979042053,
      "logps/chosen": -6.234726428985596,
      "logps/rejected": -38.455230712890625,
      "loss": 0.6432409882545471,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3129611313343048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03605951368808746,
      "rewards/margins": 2.6540074348449707,
      "rewards/rejected": -2.617948055267334,
      "step": 183,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.31437877089006355,
      "grad_norm": 3.745917320251465,
      "learning_rate": 0.0001938179101326468,
      "logits/chosen": -0.8772048354148865,
      "logits/rejected": -0.8792482614517212,
      "logps/chosen": -10.402073860168457,
      "logps/rejected": -31.16716766357422,
      "loss": 0.7786051630973816,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4461228549480438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.22984227538108826,
      "rewards/margins": 2.267395496368408,
      "rewards/rejected": -2.037553310394287,
      "step": 184,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.31608735116664,
      "grad_norm": 3.6105992794036865,
      "learning_rate": 0.00019371965372364915,
      "logits/chosen": -0.8841488361358643,
      "logits/rejected": -0.8877873420715332,
      "logps/chosen": -7.848278045654297,
      "logps/rejected": -28.794742584228516,
      "loss": 0.7686839699745178,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4148586094379425,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2514728009700775,
      "rewards/margins": 2.0990750789642334,
      "rewards/rejected": -1.8476024866104126,
      "step": 185,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3177959314432164,
      "grad_norm": 4.601129531860352,
      "learning_rate": 0.00019362064793479767,
      "logits/chosen": -0.890434980392456,
      "logits/rejected": -0.8937606811523438,
      "logps/chosen": -6.078680515289307,
      "logps/rejected": -33.81269454956055,
      "loss": 0.6938453316688538,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35300731658935547,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06243661791086197,
      "rewards/margins": 2.5140228271484375,
      "rewards/rejected": -2.4515862464904785,
      "step": 186,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3195045117197928,
      "grad_norm": 3.495896339416504,
      "learning_rate": 0.00019352089355774004,
      "logits/chosen": -0.8553616404533386,
      "logits/rejected": -0.8608652949333191,
      "logps/chosen": -5.048914432525635,
      "logps/rejected": -32.950260162353516,
      "loss": 0.5130528211593628,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27008938789367676,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2167266607284546,
      "rewards/margins": 2.171088457107544,
      "rewards/rejected": -1.9543616771697998,
      "step": 187,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.32121309199636927,
      "grad_norm": 2.0062403678894043,
      "learning_rate": 0.0001934203913901096,
      "logits/chosen": -0.8811948299407959,
      "logits/rejected": -0.8872733116149902,
      "logps/chosen": -6.987203598022461,
      "logps/rejected": -31.409507751464844,
      "loss": 0.6117798686027527,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3332153260707855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2551400661468506,
      "rewards/margins": 2.197721004486084,
      "rewards/rejected": -1.942581057548523,
      "step": 188,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3229216722729457,
      "grad_norm": 2.5417840480804443,
      "learning_rate": 0.000193319142235519,
      "logits/chosen": -0.8587099313735962,
      "logits/rejected": -0.860413670539856,
      "logps/chosen": -10.022211074829102,
      "logps/rejected": -29.925012588500977,
      "loss": 0.6710554361343384,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4114764630794525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2854654788970947,
      "rewards/margins": 2.179234743118286,
      "rewards/rejected": -1.893769383430481,
      "step": 189,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.32463025254952216,
      "grad_norm": 2.4238228797912598,
      "learning_rate": 0.0001932171469035538,
      "logits/chosen": -0.8661004900932312,
      "logits/rejected": -0.8710408806800842,
      "logps/chosen": -5.72200870513916,
      "logps/rejected": -31.211780548095703,
      "loss": 0.4626935124397278,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24239738285541534,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23706884682178497,
      "rewards/margins": 2.2727468013763428,
      "rewards/rejected": -2.0356781482696533,
      "step": 190,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.32633883282609855,
      "grad_norm": 4.148837089538574,
      "learning_rate": 0.00019311440620976597,
      "logits/chosen": -0.9142186045646667,
      "logits/rejected": -0.9197511076927185,
      "logps/chosen": -6.843296051025391,
      "logps/rejected": -34.4539680480957,
      "loss": 0.5014334321022034,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3269428014755249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.398705393075943,
      "rewards/margins": 2.5132031440734863,
      "rewards/rejected": -2.1144979000091553,
      "step": 191,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.328047413102675,
      "grad_norm": 6.930333614349365,
      "learning_rate": 0.0001930109209756673,
      "logits/chosen": -0.9393826723098755,
      "logits/rejected": -0.9431489109992981,
      "logps/chosen": -12.365182876586914,
      "logps/rejected": -36.20299530029297,
      "loss": 0.9010649919509888,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5080785155296326,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.09569638967514038,
      "rewards/margins": 2.128075361251831,
      "rewards/rejected": -2.223771810531616,
      "step": 192,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.32975599337925143,
      "grad_norm": 1.8063973188400269,
      "learning_rate": 0.00019290669202872302,
      "logits/chosen": -0.9646225571632385,
      "logits/rejected": -0.9725656509399414,
      "logps/chosen": -4.754866123199463,
      "logps/rejected": -49.72484588623047,
      "loss": 0.3587495982646942,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2268112450838089,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3622514307498932,
      "rewards/margins": 4.094050884246826,
      "rewards/rejected": -3.731799602508545,
      "step": 193,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3314645736558279,
      "grad_norm": 9.937451362609863,
      "learning_rate": 0.00019280172020234496,
      "logits/chosen": -1.0251308679580688,
      "logits/rejected": -1.030665636062622,
      "logps/chosen": -10.551065444946289,
      "logps/rejected": -53.82728576660156,
      "loss": 1.1305742263793945,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5491915345191956,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.27872776985168457,
      "rewards/margins": 3.9652843475341797,
      "rewards/rejected": -4.244011878967285,
      "step": 194,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3331731539324043,
      "grad_norm": 7.0363922119140625,
      "learning_rate": 0.00019269600633588514,
      "logits/chosen": -0.98694908618927,
      "logits/rejected": -0.9900444746017456,
      "logps/chosen": -11.45719051361084,
      "logps/rejected": -51.027496337890625,
      "loss": 0.6249094009399414,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3946501910686493,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.056097134947776794,
      "rewards/margins": 3.8215420246124268,
      "rewards/rejected": -3.8776392936706543,
      "step": 195,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3348817342089807,
      "grad_norm": 7.936800003051758,
      "learning_rate": 0.0001925895512746287,
      "logits/chosen": -0.9765729904174805,
      "logits/rejected": -0.9803340435028076,
      "logps/chosen": -7.451618194580078,
      "logps/rejected": -42.50349426269531,
      "loss": 0.8021814227104187,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49508342146873474,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0933699905872345,
      "rewards/margins": 3.030968189239502,
      "rewards/rejected": -3.124338150024414,
      "step": 196,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.33659031448555715,
      "grad_norm": 7.247754096984863,
      "learning_rate": 0.00019248235586978755,
      "logits/chosen": -0.9704450964927673,
      "logits/rejected": -0.9796913862228394,
      "logps/chosen": -5.781133651733398,
      "logps/rejected": -43.08665084838867,
      "loss": 0.5009433627128601,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3090832829475403,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17386135458946228,
      "rewards/margins": 3.0037147998809814,
      "rewards/rejected": -2.8298535346984863,
      "step": 197,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3382988947621336,
      "grad_norm": 4.252370357513428,
      "learning_rate": 0.0001923744209784933,
      "logits/chosen": -0.9834634065628052,
      "logits/rejected": -0.9903834462165833,
      "logps/chosen": -10.431640625,
      "logps/rejected": -41.34126663208008,
      "loss": 0.892498791217804,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5820133686065674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0897313803434372,
      "rewards/margins": 2.829003095626831,
      "rewards/rejected": -2.739271640777588,
      "step": 198,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.34000747503871004,
      "grad_norm": 7.74161958694458,
      "learning_rate": 0.00019226574746379038,
      "logits/chosen": -0.9264886975288391,
      "logits/rejected": -0.9303056597709656,
      "logps/chosen": -18.308433532714844,
      "logps/rejected": -34.25849914550781,
      "loss": 1.4104645252227783,
      "memory(GiB)": 46.38,
      "nll_loss": 1.1379212141036987,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23661470413208008,
      "rewards/margins": 2.288109540939331,
      "rewards/rejected": -2.051494598388672,
      "step": 199,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3417160553152864,
      "grad_norm": 19.06857681274414,
      "learning_rate": 0.00019215633619462942,
      "logits/chosen": -0.9472784399986267,
      "logits/rejected": -0.9554142951965332,
      "logps/chosen": -9.5065336227417,
      "logps/rejected": -40.612945556640625,
      "loss": 1.1060166358947754,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7597510814666748,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.030025824904441833,
      "rewards/margins": 2.3827638626098633,
      "rewards/rejected": -2.35273814201355,
      "step": 200,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.34342463559186287,
      "grad_norm": 3.042719841003418,
      "learning_rate": 0.00019204618804585994,
      "logits/chosen": -0.8972377181053162,
      "logits/rejected": -0.9024269580841064,
      "logps/chosen": -6.98057222366333,
      "logps/rejected": -34.55181884765625,
      "loss": 0.5098893642425537,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2953639328479767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31456321477890015,
      "rewards/margins": 2.592310905456543,
      "rewards/rejected": -2.277747392654419,
      "step": 201,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3451332158684393,
      "grad_norm": 3.191251516342163,
      "learning_rate": 0.00019193530389822363,
      "logits/chosen": -0.9289453625679016,
      "logits/rejected": -0.9359845519065857,
      "logps/chosen": -9.659342765808105,
      "logps/rejected": -37.61637878417969,
      "loss": 1.0019766092300415,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6679946780204773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1197037547826767,
      "rewards/margins": 2.4200377464294434,
      "rewards/rejected": -2.3003342151641846,
      "step": 202,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.34684179614501576,
      "grad_norm": 4.930737018585205,
      "learning_rate": 0.00019182368463834722,
      "logits/chosen": -0.8630688190460205,
      "logits/rejected": -0.8667917847633362,
      "logps/chosen": -9.344999313354492,
      "logps/rejected": -36.20460510253906,
      "loss": 0.5554157495498657,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36027616262435913,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1459665298461914,
      "rewards/margins": 2.5479657649993896,
      "rewards/rejected": -2.4019992351531982,
      "step": 203,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3485503764215922,
      "grad_norm": 2.625263214111328,
      "learning_rate": 0.00019171133115873526,
      "logits/chosen": -0.8702833652496338,
      "logits/rejected": -0.8725969195365906,
      "logps/chosen": -7.262643814086914,
      "logps/rejected": -34.7918701171875,
      "loss": 0.4979276657104492,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3470659554004669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23693254590034485,
      "rewards/margins": 2.7716078758239746,
      "rewards/rejected": -2.534675359725952,
      "step": 204,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3502589566981686,
      "grad_norm": 4.522634029388428,
      "learning_rate": 0.0001915982443577632,
      "logits/chosen": -0.8535888195037842,
      "logits/rejected": -0.859410285949707,
      "logps/chosen": -6.171905517578125,
      "logps/rejected": -37.568458557128906,
      "loss": 0.5628905296325684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3058054447174072,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09114877134561539,
      "rewards/margins": 2.7372331619262695,
      "rewards/rejected": -2.6460840702056885,
      "step": 205,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.35196753697474503,
      "grad_norm": 3.1796517372131348,
      "learning_rate": 0.00019148442513967014,
      "logits/chosen": -0.8468549251556396,
      "logits/rejected": -0.8539971709251404,
      "logps/chosen": -9.5682954788208,
      "logps/rejected": -36.71331024169922,
      "loss": 0.6788620948791504,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4944525957107544,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10999859869480133,
      "rewards/margins": 2.5533294677734375,
      "rewards/rejected": -2.443330764770508,
      "step": 206,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3536761172513215,
      "grad_norm": 3.1952710151672363,
      "learning_rate": 0.00019136987441455144,
      "logits/chosen": -0.8667152523994446,
      "logits/rejected": -0.8705443143844604,
      "logps/chosen": -9.689000129699707,
      "logps/rejected": -30.918254852294922,
      "loss": 0.6780470013618469,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41608238220214844,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2547697126865387,
      "rewards/margins": 2.3299858570098877,
      "rewards/rejected": -2.075216293334961,
      "step": 207,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3553846975278979,
      "grad_norm": 2.4894015789031982,
      "learning_rate": 0.00019125459309835173,
      "logits/chosen": -0.8714242577552795,
      "logits/rejected": -0.8752931356430054,
      "logps/chosen": -7.453229904174805,
      "logps/rejected": -31.325904846191406,
      "loss": 0.6907439827919006,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4350113868713379,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.17801815271377563,
      "rewards/margins": 2.264295816421509,
      "rewards/rejected": -2.086277484893799,
      "step": 208,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.35709327780447436,
      "grad_norm": 3.2602062225341797,
      "learning_rate": 0.00019113858211285725,
      "logits/chosen": -0.872046709060669,
      "logits/rejected": -0.8767064213752747,
      "logps/chosen": -4.748498439788818,
      "logps/rejected": -36.4671745300293,
      "loss": 0.47542524337768555,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24451857805252075,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20409636199474335,
      "rewards/margins": 2.852433443069458,
      "rewards/rejected": -2.6483371257781982,
      "step": 209,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.35880185808105075,
      "grad_norm": 2.723602533340454,
      "learning_rate": 0.00019102184238568878,
      "logits/chosen": -0.872526228427887,
      "logits/rejected": -0.8720419406890869,
      "logps/chosen": -9.495769500732422,
      "logps/rejected": -35.4581184387207,
      "loss": 0.6998460292816162,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4141753911972046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08976475894451141,
      "rewards/margins": 2.3428900241851807,
      "rewards/rejected": -2.2531251907348633,
      "step": 210,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3605104383576272,
      "grad_norm": 7.090069770812988,
      "learning_rate": 0.00019090437485029413,
      "logits/chosen": -0.8751546740531921,
      "logits/rejected": -0.8804129362106323,
      "logps/chosen": -5.539629936218262,
      "logps/rejected": -41.93748092651367,
      "loss": 0.414511114358902,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24404364824295044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2657608985900879,
      "rewards/margins": 3.283001184463501,
      "rewards/rejected": -3.017240524291992,
      "step": 211,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.36221901863420364,
      "grad_norm": 3.3576340675354004,
      "learning_rate": 0.00019078618044594052,
      "logits/chosen": -0.9006087779998779,
      "logits/rejected": -0.9030022621154785,
      "logps/chosen": -7.570390224456787,
      "logps/rejected": -34.19873046875,
      "loss": 0.4811999499797821,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29778629541397095,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24354203045368195,
      "rewards/margins": 2.750795364379883,
      "rewards/rejected": -2.5072531700134277,
      "step": 212,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3639275989107801,
      "grad_norm": 4.36439323425293,
      "learning_rate": 0.00019066726011770726,
      "logits/chosen": -0.8826490640640259,
      "logits/rejected": -0.8854631185531616,
      "logps/chosen": -7.964271068572998,
      "logps/rejected": -36.53171920776367,
      "loss": 0.7800939679145813,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4539424479007721,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1163502037525177,
      "rewards/margins": 2.651106357574463,
      "rewards/rejected": -2.5347559452056885,
      "step": 213,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3656361791873565,
      "grad_norm": 3.075863838195801,
      "learning_rate": 0.00019054761481647817,
      "logits/chosen": -0.9236173033714294,
      "logits/rejected": -0.9311164021492004,
      "logps/chosen": -5.548548698425293,
      "logps/rejected": -39.62729263305664,
      "loss": 0.5424690842628479,
      "memory(GiB)": 46.38,
      "nll_loss": 0.37887126207351685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2379705011844635,
      "rewards/margins": 2.8928284645080566,
      "rewards/rejected": -2.654857873916626,
      "step": 214,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3673447594639329,
      "grad_norm": 4.081761360168457,
      "learning_rate": 0.0001904272454989338,
      "logits/chosen": -0.9173122644424438,
      "logits/rejected": -0.9237719774246216,
      "logps/chosen": -7.586677551269531,
      "logps/rejected": -39.44381332397461,
      "loss": 0.7125780582427979,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4088558554649353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10065207630395889,
      "rewards/margins": 2.89681077003479,
      "rewards/rejected": -2.796158790588379,
      "step": 215,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.36905333974050936,
      "grad_norm": 6.38205099105835,
      "learning_rate": 0.00019030615312754412,
      "logits/chosen": -0.8755264282226562,
      "logits/rejected": -0.8772911429405212,
      "logps/chosen": -10.058317184448242,
      "logps/rejected": -29.399351119995117,
      "loss": 0.6085036993026733,
      "memory(GiB)": 46.38,
      "nll_loss": 0.37356507778167725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.257190465927124,
      "rewards/margins": 2.3912007808685303,
      "rewards/rejected": -2.1340105533599854,
      "step": 216,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3707619200170858,
      "grad_norm": 3.965923547744751,
      "learning_rate": 0.00019018433867056043,
      "logits/chosen": -0.8582707047462463,
      "logits/rejected": -0.8643903136253357,
      "logps/chosen": -7.274492263793945,
      "logps/rejected": -38.09757995605469,
      "loss": 0.7568970918655396,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5492669343948364,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09097186475992203,
      "rewards/margins": 2.650477170944214,
      "rewards/rejected": -2.5595054626464844,
      "step": 217,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.37247050029366224,
      "grad_norm": 6.908701419830322,
      "learning_rate": 0.00019006180310200788,
      "logits/chosen": -0.8987607955932617,
      "logits/rejected": -0.9025367498397827,
      "logps/chosen": -9.353863716125488,
      "logps/rejected": -41.96833801269531,
      "loss": 0.5579907298088074,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32021838426589966,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06166917830705643,
      "rewards/margins": 3.1308600902557373,
      "rewards/rejected": -3.069190740585327,
      "step": 218,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3741790805702387,
      "grad_norm": 11.417703628540039,
      "learning_rate": 0.00018993854740167768,
      "logits/chosen": -0.8815793991088867,
      "logits/rejected": -0.8841702342033386,
      "logps/chosen": -8.630674362182617,
      "logps/rejected": -32.739219665527344,
      "loss": 0.7540084719657898,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42135101556777954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.017735840752720833,
      "rewards/margins": 2.403879165649414,
      "rewards/rejected": -2.386143445968628,
      "step": 219,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3758876608468151,
      "grad_norm": 7.359007358551025,
      "learning_rate": 0.00018981457255511902,
      "logits/chosen": -0.8066080212593079,
      "logits/rejected": -0.8113219141960144,
      "logps/chosen": -10.087733268737793,
      "logps/rejected": -34.230953216552734,
      "loss": 0.7611066102981567,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38534992933273315,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18794448673725128,
      "rewards/margins": 2.276632785797119,
      "rewards/rejected": -2.088688373565674,
      "step": 220,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3775962411233915,
      "grad_norm": 2.5772690773010254,
      "learning_rate": 0.00018968987955363153,
      "logits/chosen": -0.7865528464317322,
      "logits/rejected": -0.7891740202903748,
      "logps/chosen": -13.03406810760498,
      "logps/rejected": -25.763931274414062,
      "loss": 0.8478187918663025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5038392543792725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3180733621120453,
      "rewards/margins": 1.6984466314315796,
      "rewards/rejected": -1.3803733587265015,
      "step": 221,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.37930482139996796,
      "grad_norm": 3.7778072357177734,
      "learning_rate": 0.00018956446939425716,
      "logits/chosen": -0.826096773147583,
      "logits/rejected": -0.8322123885154724,
      "logps/chosen": -9.263298988342285,
      "logps/rejected": -25.442668914794922,
      "loss": 0.6595274209976196,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3389233946800232,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24548465013504028,
      "rewards/margins": 1.5120353698730469,
      "rewards/rejected": -1.2665507793426514,
      "step": 222,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3810134016765444,
      "grad_norm": 4.857659339904785,
      "learning_rate": 0.00018943834307977216,
      "logits/chosen": -0.8075899481773376,
      "logits/rejected": -0.8120123744010925,
      "logps/chosen": -4.7845778465271,
      "logps/rejected": -21.904151916503906,
      "loss": 0.6341399550437927,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2501222789287567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19457918405532837,
      "rewards/margins": 1.1304917335510254,
      "rewards/rejected": -0.9359126091003418,
      "step": 223,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.38272198195312085,
      "grad_norm": 2.3448665142059326,
      "learning_rate": 0.00018931150161867916,
      "logits/chosen": -0.882806658744812,
      "logits/rejected": -0.8879884481430054,
      "logps/chosen": -9.03802490234375,
      "logps/rejected": -24.53693199157715,
      "loss": 0.7897974848747253,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38777855038642883,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1946476548910141,
      "rewards/margins": 1.5893993377685547,
      "rewards/rejected": -1.3947515487670898,
      "step": 224,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.38443056222969724,
      "grad_norm": 11.419641494750977,
      "learning_rate": 0.00018918394602519914,
      "logits/chosen": -0.8994475603103638,
      "logits/rejected": -0.9015949964523315,
      "logps/chosen": -7.98261022567749,
      "logps/rejected": -27.965211868286133,
      "loss": 0.7503970265388489,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4178537130355835,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.22919557988643646,
      "rewards/margins": 1.8219597339630127,
      "rewards/rejected": -1.592764139175415,
      "step": 225,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3861391425062737,
      "grad_norm": 5.716240406036377,
      "learning_rate": 0.00018905567731926324,
      "logits/chosen": -0.8648837208747864,
      "logits/rejected": -0.8741611838340759,
      "logps/chosen": -6.251923084259033,
      "logps/rejected": -36.683223724365234,
      "loss": 0.5255903601646423,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3399491310119629,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2363051176071167,
      "rewards/margins": 2.3525753021240234,
      "rewards/rejected": -2.1162703037261963,
      "step": 226,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.3878477227828501,
      "grad_norm": 2.4492273330688477,
      "learning_rate": 0.00018892669652650456,
      "logits/chosen": -0.8707509636878967,
      "logits/rejected": -0.8740925192832947,
      "logps/chosen": -8.421378135681152,
      "logps/rejected": -34.801605224609375,
      "loss": 0.6341971158981323,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4148554801940918,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3119761645793915,
      "rewards/margins": 2.5685765743255615,
      "rewards/rejected": -2.2566003799438477,
      "step": 227,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.38955630305942657,
      "grad_norm": 7.605607986450195,
      "learning_rate": 0.00018879700467825013,
      "logits/chosen": -0.8900420069694519,
      "logits/rejected": -0.8933430910110474,
      "logps/chosen": -5.360377311706543,
      "logps/rejected": -33.13306427001953,
      "loss": 0.4900038540363312,
      "memory(GiB)": 46.38,
      "nll_loss": 0.28295254707336426,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3359947204589844,
      "rewards/margins": 2.5908119678497314,
      "rewards/rejected": -2.254817008972168,
      "step": 228,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.391264883336003,
      "grad_norm": 8.061660766601562,
      "learning_rate": 0.00018866660281151247,
      "logits/chosen": -0.9369041919708252,
      "logits/rejected": -0.9415433406829834,
      "logps/chosen": -4.601605415344238,
      "logps/rejected": -35.486732482910156,
      "loss": 0.5184635519981384,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2989267408847809,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22662585973739624,
      "rewards/margins": 2.7763917446136475,
      "rewards/rejected": -2.5497660636901855,
      "step": 229,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3929734636125794,
      "grad_norm": 3.7317209243774414,
      "learning_rate": 0.0001885354919689815,
      "logits/chosen": -0.9507468342781067,
      "logits/rejected": -0.9519726634025574,
      "logps/chosen": -6.97619104385376,
      "logps/rejected": -38.649261474609375,
      "loss": 0.4864623546600342,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2420811951160431,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16122400760650635,
      "rewards/margins": 2.8739140033721924,
      "rewards/rejected": -2.7126896381378174,
      "step": 230,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.39468204388915584,
      "grad_norm": 2.446930408477783,
      "learning_rate": 0.00018840367319901592,
      "logits/chosen": -0.9463286399841309,
      "logits/rejected": -0.9530838131904602,
      "logps/chosen": -4.321158409118652,
      "logps/rejected": -45.260074615478516,
      "loss": 0.2898697853088379,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19011972844600677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2383752316236496,
      "rewards/margins": 3.6623451709747314,
      "rewards/rejected": -3.4239697456359863,
      "step": 231,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3963906241657323,
      "grad_norm": 10.94664478302002,
      "learning_rate": 0.0001882711475556352,
      "logits/chosen": -0.9894860982894897,
      "logits/rejected": -0.9986059665679932,
      "logps/chosen": -7.955844879150391,
      "logps/rejected": -57.02360534667969,
      "loss": 1.0545183420181274,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7007216215133667,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0017464309930801392,
      "rewards/margins": 4.065786361694336,
      "rewards/rejected": -4.067533016204834,
      "step": 232,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.39809920444230873,
      "grad_norm": 5.375171661376953,
      "learning_rate": 0.0001881379160985108,
      "logits/chosen": -1.0208549499511719,
      "logits/rejected": -1.0248205661773682,
      "logps/chosen": -7.056022644042969,
      "logps/rejected": -52.94482421875,
      "loss": 0.6524523496627808,
      "memory(GiB)": 46.38,
      "nll_loss": 0.44853687286376953,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08947555720806122,
      "rewards/margins": 4.471720218658447,
      "rewards/rejected": -4.382244110107422,
      "step": 233,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.3998077847188852,
      "grad_norm": 12.273642539978027,
      "learning_rate": 0.00018800397989295794,
      "logits/chosen": -0.9972729682922363,
      "logits/rejected": -1.0022565126419067,
      "logps/chosen": -10.15302848815918,
      "logps/rejected": -46.114967346191406,
      "loss": 0.7069646716117859,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4566623270511627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1436801254749298,
      "rewards/margins": 3.605038642883301,
      "rewards/rejected": -3.4613585472106934,
      "step": 234,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.40151636499546156,
      "grad_norm": 8.246232986450195,
      "learning_rate": 0.00018786934000992688,
      "logits/chosen": -1.0225094556808472,
      "logits/rejected": -1.019821286201477,
      "logps/chosen": -11.058813095092773,
      "logps/rejected": -53.49971008300781,
      "loss": 0.811690092086792,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42450016736984253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12926799058914185,
      "rewards/margins": 4.236933708190918,
      "rewards/rejected": -4.366201877593994,
      "step": 235,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.403224945272038,
      "grad_norm": 13.076598167419434,
      "learning_rate": 0.00018773399752599463,
      "logits/chosen": -1.001189112663269,
      "logits/rejected": -1.0065340995788574,
      "logps/chosen": -8.285541534423828,
      "logps/rejected": -48.76884460449219,
      "loss": 0.6642578840255737,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3762781023979187,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.013312354683876038,
      "rewards/margins": 3.655507802963257,
      "rewards/rejected": -3.642195701599121,
      "step": 236,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.40493352554861445,
      "grad_norm": 3.6171457767486572,
      "learning_rate": 0.000187597953523356,
      "logits/chosen": -0.9681710600852966,
      "logits/rejected": -0.9759187698364258,
      "logps/chosen": -8.311320304870605,
      "logps/rejected": -54.762943267822266,
      "loss": 0.5605093240737915,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39821937680244446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.021861745044589043,
      "rewards/margins": 4.216716289520264,
      "rewards/rejected": -4.194854736328125,
      "step": 237,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4066421058251909,
      "grad_norm": 15.052586555480957,
      "learning_rate": 0.00018746120908981527,
      "logits/chosen": -0.9332698583602905,
      "logits/rejected": -0.9356809258460999,
      "logps/chosen": -6.916642189025879,
      "logps/rejected": -39.75300979614258,
      "loss": 0.5308848023414612,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33774426579475403,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11101798713207245,
      "rewards/margins": 3.2900469303131104,
      "rewards/rejected": -3.1790287494659424,
      "step": 238,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.40835068610176734,
      "grad_norm": 1.4588333368301392,
      "learning_rate": 0.00018732376531877725,
      "logits/chosen": -0.8663907647132874,
      "logits/rejected": -0.8769683241844177,
      "logps/chosen": -7.994008541107178,
      "logps/rejected": -53.477638244628906,
      "loss": 0.3602023124694824,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27253228425979614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39557039737701416,
      "rewards/margins": 4.195072174072266,
      "rewards/rejected": -3.799501895904541,
      "step": 239,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.4100592663783437,
      "grad_norm": 2.3394670486450195,
      "learning_rate": 0.0001871856233092387,
      "logits/chosen": -0.8316817283630371,
      "logits/rejected": -0.8333123326301575,
      "logps/chosen": -9.785093307495117,
      "logps/rejected": -40.74665069580078,
      "loss": 0.4092681407928467,
      "memory(GiB)": 46.38,
      "nll_loss": 0.260936975479126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18386238813400269,
      "rewards/margins": 2.943046808242798,
      "rewards/rejected": -2.7591843605041504,
      "step": 240,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.41176784665492017,
      "grad_norm": 15.97940731048584,
      "learning_rate": 0.00018704678416577941,
      "logits/chosen": -0.8120774030685425,
      "logits/rejected": -0.8134472370147705,
      "logps/chosen": -11.521053314208984,
      "logps/rejected": -35.660865783691406,
      "loss": 0.8933948874473572,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5571330785751343,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0759764090180397,
      "rewards/margins": 2.2313168048858643,
      "rewards/rejected": -2.30729341506958,
      "step": 241,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.4134764269314966,
      "grad_norm": 4.722672462463379,
      "learning_rate": 0.00018690724899855348,
      "logits/chosen": -0.7959482669830322,
      "logits/rejected": -0.7984044551849365,
      "logps/chosen": -5.855531692504883,
      "logps/rejected": -40.35407257080078,
      "loss": 0.49105608463287354,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2971791923046112,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.221318781375885,
      "rewards/margins": 3.0078468322753906,
      "rewards/rejected": -2.7865281105041504,
      "step": 242,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.41518500720807305,
      "grad_norm": 6.576427936553955,
      "learning_rate": 0.00018676701892328046,
      "logits/chosen": -0.7806111574172974,
      "logits/rejected": -0.783092200756073,
      "logps/chosen": -4.328828811645508,
      "logps/rejected": -40.24433517456055,
      "loss": 0.45146334171295166,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30311349034309387,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22997835278511047,
      "rewards/margins": 3.151667833328247,
      "rewards/rejected": -2.921689510345459,
      "step": 243,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4168935874846495,
      "grad_norm": 3.0237584114074707,
      "learning_rate": 0.0001866260950612362,
      "logits/chosen": -0.768136203289032,
      "logits/rejected": -0.7717478275299072,
      "logps/chosen": -4.951301097869873,
      "logps/rejected": -33.46363830566406,
      "loss": 0.4757350981235504,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20984753966331482,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.28256160020828247,
      "rewards/margins": 2.4234559535980225,
      "rewards/rejected": -2.140894651412964,
      "step": 244,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4186021677612259,
      "grad_norm": 6.6580657958984375,
      "learning_rate": 0.0001864844785392442,
      "logits/chosen": -0.745742917060852,
      "logits/rejected": -0.7464924454689026,
      "logps/chosen": -10.95466136932373,
      "logps/rejected": -44.5806999206543,
      "loss": 0.5750457048416138,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3861500322818756,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1691206991672516,
      "rewards/margins": 3.562136650085449,
      "rewards/rejected": -3.3930160999298096,
      "step": 245,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.42031074803780233,
      "grad_norm": 6.422461032867432,
      "learning_rate": 0.00018634217048966637,
      "logits/chosen": -0.7587261199951172,
      "logits/rejected": -0.7620642185211182,
      "logps/chosen": -6.3410139083862305,
      "logps/rejected": -33.94182586669922,
      "loss": 0.6545197367668152,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30830273032188416,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06438297033309937,
      "rewards/margins": 2.2670321464538574,
      "rewards/rejected": -2.3314151763916016,
      "step": 246,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.42201932831437877,
      "grad_norm": 4.497966289520264,
      "learning_rate": 0.00018619917205039407,
      "logits/chosen": -0.7642217874526978,
      "logits/rejected": -0.7698625326156616,
      "logps/chosen": -6.284250259399414,
      "logps/rejected": -46.83761215209961,
      "loss": 0.4105835258960724,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2564704418182373,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22518175840377808,
      "rewards/margins": 3.5758605003356934,
      "rewards/rejected": -3.3506786823272705,
      "step": 247,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4237279085909552,
      "grad_norm": 3.6431024074554443,
      "learning_rate": 0.00018605548436483906,
      "logits/chosen": -0.7721464037895203,
      "logits/rejected": -0.7743059396743774,
      "logps/chosen": -8.004281044006348,
      "logps/rejected": -34.3867073059082,
      "loss": 0.6020833849906921,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3395420014858246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2702520787715912,
      "rewards/margins": 2.563030242919922,
      "rewards/rejected": -2.292778491973877,
      "step": 248,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.42543648886753166,
      "grad_norm": 2.3539841175079346,
      "learning_rate": 0.0001859111085819242,
      "logits/chosen": -0.7793521285057068,
      "logits/rejected": -0.7824581265449524,
      "logps/chosen": -12.44356918334961,
      "logps/rejected": -37.551902770996094,
      "loss": 0.569586455821991,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4044492542743683,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30807700753211975,
      "rewards/margins": 2.927927017211914,
      "rewards/rejected": -2.6198501586914062,
      "step": 249,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.42714506914410805,
      "grad_norm": 3.378519058227539,
      "learning_rate": 0.00018576604585607441,
      "logits/chosen": -0.7945507168769836,
      "logits/rejected": -0.7992746233940125,
      "logps/chosen": -7.014500141143799,
      "logps/rejected": -46.093238830566406,
      "loss": 0.47747644782066345,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2733898162841797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10495103895664215,
      "rewards/margins": 3.2655720710754395,
      "rewards/rejected": -3.160621166229248,
      "step": 250,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.4288536494206845,
      "grad_norm": 4.231056213378906,
      "learning_rate": 0.00018562029734720743,
      "logits/chosen": -0.8051905632019043,
      "logits/rejected": -0.8092507123947144,
      "logps/chosen": -6.725405693054199,
      "logps/rejected": -37.768829345703125,
      "loss": 0.5110489726066589,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31649249792099,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19485878944396973,
      "rewards/margins": 2.8023769855499268,
      "rewards/rejected": -2.607518196105957,
      "step": 251,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.43056222969726093,
      "grad_norm": 11.981834411621094,
      "learning_rate": 0.00018547386422072438,
      "logits/chosen": -0.7716567516326904,
      "logits/rejected": -0.773201584815979,
      "logps/chosen": -9.067695617675781,
      "logps/rejected": -36.645240783691406,
      "loss": 0.8504749536514282,
      "memory(GiB)": 46.38,
      "nll_loss": 0.514085054397583,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07366381585597992,
      "rewards/margins": 2.7288200855255127,
      "rewards/rejected": -2.8024840354919434,
      "step": 252,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4322708099738374,
      "grad_norm": 8.43531608581543,
      "learning_rate": 0.0001853267476475007,
      "logits/chosen": -0.8169345855712891,
      "logits/rejected": -0.8159929513931274,
      "logps/chosen": -11.076251029968262,
      "logps/rejected": -42.98237991333008,
      "loss": 0.7847392559051514,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5229505300521851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1810646504163742,
      "rewards/margins": 3.4308125972747803,
      "rewards/rejected": -3.2497482299804688,
      "step": 253,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4339793902504138,
      "grad_norm": 6.133362770080566,
      "learning_rate": 0.0001851789488038765,
      "logits/chosen": -0.7999134063720703,
      "logits/rejected": -0.8003981709480286,
      "logps/chosen": -6.108333587646484,
      "logps/rejected": -41.47050857543945,
      "loss": 0.6103618741035461,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3681967258453369,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1454550325870514,
      "rewards/margins": 3.3938965797424316,
      "rewards/rejected": -3.248441457748413,
      "step": 254,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4356879705269902,
      "grad_norm": 27.34607696533203,
      "learning_rate": 0.0001850304688716475,
      "logits/chosen": -0.7655246257781982,
      "logits/rejected": -0.7673658132553101,
      "logps/chosen": -9.124423027038574,
      "logps/rejected": -40.40342330932617,
      "loss": 0.9848135709762573,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6617517471313477,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10712965577840805,
      "rewards/margins": 2.936744213104248,
      "rewards/rejected": -2.8296148777008057,
      "step": 255,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.43739655080356665,
      "grad_norm": 3.467927932739258,
      "learning_rate": 0.00018488130903805523,
      "logits/chosen": -0.7815067768096924,
      "logits/rejected": -0.7830615639686584,
      "logps/chosen": -6.11367654800415,
      "logps/rejected": -43.792381286621094,
      "loss": 0.3926101326942444,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2346831113100052,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3161233365535736,
      "rewards/margins": 3.5978031158447266,
      "rewards/rejected": -3.281679630279541,
      "step": 256,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4391051310801431,
      "grad_norm": 3.6716325283050537,
      "learning_rate": 0.00018473147049577774,
      "logits/chosen": -0.8119379878044128,
      "logits/rejected": -0.8167931437492371,
      "logps/chosen": -5.306201934814453,
      "logps/rejected": -48.548152923583984,
      "loss": 0.3491962254047394,
      "memory(GiB)": 46.38,
      "nll_loss": 0.214007169008255,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19937750697135925,
      "rewards/margins": 3.6415648460388184,
      "rewards/rejected": -3.442187547683716,
      "step": 257,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.44081371135671954,
      "grad_norm": 25.686939239501953,
      "learning_rate": 0.00018458095444292004,
      "logits/chosen": -0.825552761554718,
      "logits/rejected": -0.8325546383857727,
      "logps/chosen": -5.261894702911377,
      "logps/rejected": -62.226497650146484,
      "loss": 0.434018075466156,
      "memory(GiB)": 46.38,
      "nll_loss": 0.287321001291275,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32799047231674194,
      "rewards/margins": 4.790035247802734,
      "rewards/rejected": -4.462045192718506,
      "step": 258,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.442522291633296,
      "grad_norm": 6.754378318786621,
      "learning_rate": 0.00018442976208300448,
      "logits/chosen": -0.8270843029022217,
      "logits/rejected": -0.8281224966049194,
      "logps/chosen": -8.492132186889648,
      "logps/rejected": -36.23853302001953,
      "loss": 0.6711763739585876,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3068382143974304,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05488736927509308,
      "rewards/margins": 2.777843952178955,
      "rewards/rejected": -2.722956418991089,
      "step": 259,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.44423087190987237,
      "grad_norm": 4.042886257171631,
      "learning_rate": 0.00018427789462496113,
      "logits/chosen": -0.8525445461273193,
      "logits/rejected": -0.8591835498809814,
      "logps/chosen": -8.231825828552246,
      "logps/rejected": -47.20307540893555,
      "loss": 0.8233700394630432,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5154776573181152,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.00042735040187835693,
      "rewards/margins": 3.4719269275665283,
      "rewards/rejected": -3.4723546504974365,
      "step": 260,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4459394521864488,
      "grad_norm": 12.385782241821289,
      "learning_rate": 0.00018412535328311814,
      "logits/chosen": -0.822756290435791,
      "logits/rejected": -0.8276916742324829,
      "logps/chosen": -7.529094219207764,
      "logps/rejected": -39.26826477050781,
      "loss": 0.7842540740966797,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41948723793029785,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.09604156017303467,
      "rewards/margins": 2.7830724716186523,
      "rewards/rejected": -2.8791141510009766,
      "step": 261,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.44764803246302526,
      "grad_norm": 2.720581531524658,
      "learning_rate": 0.00018397213927719199,
      "logits/chosen": -0.7854418158531189,
      "logits/rejected": -0.7872219085693359,
      "logps/chosen": -7.754611968994141,
      "logps/rejected": -47.40762710571289,
      "loss": 0.36977341771125793,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23527176678180695,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1266416311264038,
      "rewards/margins": 3.6666605472564697,
      "rewards/rejected": -3.5400190353393555,
      "step": 262,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4493566127396017,
      "grad_norm": 3.862173557281494,
      "learning_rate": 0.00018381825383227782,
      "logits/chosen": -0.8271799087524414,
      "logits/rejected": -0.831858217716217,
      "logps/chosen": -6.855618476867676,
      "logps/rejected": -46.51610565185547,
      "loss": 0.4688463509082794,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3772401511669159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2653267979621887,
      "rewards/margins": 3.657702922821045,
      "rewards/rejected": -3.392376184463501,
      "step": 263,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.45106519301617815,
      "grad_norm": 2.1108744144439697,
      "learning_rate": 0.00018366369817883954,
      "logits/chosen": -0.8001513481140137,
      "logits/rejected": -0.8060557842254639,
      "logps/chosen": -5.957093715667725,
      "logps/rejected": -49.00318908691406,
      "loss": 0.42761629819869995,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2863171100616455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.46388381719589233,
      "rewards/margins": 3.825634002685547,
      "rewards/rejected": -3.3617501258850098,
      "step": 264,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.45277377329275453,
      "grad_norm": 4.1208696365356445,
      "learning_rate": 0.00018350847355270004,
      "logits/chosen": -0.7904451489448547,
      "logits/rejected": -0.7935886979103088,
      "logps/chosen": -12.136712074279785,
      "logps/rejected": -48.772483825683594,
      "loss": 0.6188664436340332,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4114742577075958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2880480885505676,
      "rewards/margins": 3.680865526199341,
      "rewards/rejected": -3.392817258834839,
      "step": 265,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.454482353569331,
      "grad_norm": 5.501693248748779,
      "learning_rate": 0.00018335258119503127,
      "logits/chosen": -0.8123242259025574,
      "logits/rejected": -0.8159137964248657,
      "logps/chosen": -6.165188789367676,
      "logps/rejected": -47.05010986328125,
      "loss": 0.45226797461509705,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35374370217323303,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20915599167346954,
      "rewards/margins": 3.658740997314453,
      "rewards/rejected": -3.4495849609375,
      "step": 266,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4561909338459074,
      "grad_norm": 11.674309730529785,
      "learning_rate": 0.00018319602235234442,
      "logits/chosen": -0.7954471111297607,
      "logits/rejected": -0.7979443073272705,
      "logps/chosen": -9.799958229064941,
      "logps/rejected": -51.7642822265625,
      "loss": 0.7218329310417175,
      "memory(GiB)": 46.38,
      "nll_loss": 0.44642627239227295,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.12691040337085724,
      "rewards/margins": 3.8264598846435547,
      "rewards/rejected": -3.9533700942993164,
      "step": 267,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.45789951412248386,
      "grad_norm": 21.236757278442383,
      "learning_rate": 0.00018303879827647975,
      "logits/chosen": -0.7900252938270569,
      "logits/rejected": -0.7937359809875488,
      "logps/chosen": -9.592187881469727,
      "logps/rejected": -43.07557678222656,
      "loss": 0.6332522034645081,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5019268989562988,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1493360996246338,
      "rewards/margins": 3.395822763442993,
      "rewards/rejected": -3.2464866638183594,
      "step": 268,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4596080943990603,
      "grad_norm": 13.447307586669922,
      "learning_rate": 0.00018288091022459685,
      "logits/chosen": -0.7903649210929871,
      "logits/rejected": -0.792717695236206,
      "logps/chosen": -8.880897521972656,
      "logps/rejected": -48.42265701293945,
      "loss": 1.129546046257019,
      "memory(GiB)": 46.38,
      "nll_loss": 0.9457821249961853,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.0744566023349762,
      "rewards/margins": 3.6476986408233643,
      "rewards/rejected": -3.7221555709838867,
      "step": 269,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4613166746756367,
      "grad_norm": 15.565564155578613,
      "learning_rate": 0.00018272235945916436,
      "logits/chosen": -0.7472554445266724,
      "logits/rejected": -0.7549644708633423,
      "logps/chosen": -13.44924259185791,
      "logps/rejected": -50.098731994628906,
      "loss": 1.1531708240509033,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8039681911468506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14791038632392883,
      "rewards/margins": 3.323791980743408,
      "rewards/rejected": -3.4717020988464355,
      "step": 270,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.46302525495221314,
      "grad_norm": 12.007596015930176,
      "learning_rate": 0.00018256314724794992,
      "logits/chosen": -0.8131598234176636,
      "logits/rejected": -0.8223381638526917,
      "logps/chosen": -5.585975646972656,
      "logps/rejected": -48.965389251708984,
      "loss": 0.4150363802909851,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2882520854473114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21873319149017334,
      "rewards/margins": 3.403482675552368,
      "rewards/rejected": -3.1847496032714844,
      "step": 271,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4647338352287896,
      "grad_norm": 16.59933853149414,
      "learning_rate": 0.00018240327486401022,
      "logits/chosen": -0.8030964136123657,
      "logits/rejected": -0.8079983592033386,
      "logps/chosen": -8.950094223022461,
      "logps/rejected": -47.7308349609375,
      "loss": 0.9466665387153625,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6910676956176758,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.06938548386096954,
      "rewards/margins": 3.2243666648864746,
      "rewards/rejected": -3.293752431869507,
      "step": 272,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.466442415505366,
      "grad_norm": 4.702775955200195,
      "learning_rate": 0.00018224274358568052,
      "logits/chosen": -0.8188191652297974,
      "logits/rejected": -0.8212082982063293,
      "logps/chosen": -7.591650485992432,
      "logps/rejected": -37.239356994628906,
      "loss": 0.5474280714988708,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3606598973274231,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.29892435669898987,
      "rewards/margins": 3.0769505500793457,
      "rewards/rejected": -2.7780261039733887,
      "step": 273,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.46815099578194247,
      "grad_norm": 27.876619338989258,
      "learning_rate": 0.00018208155469656462,
      "logits/chosen": -0.8266133666038513,
      "logits/rejected": -0.8309741020202637,
      "logps/chosen": -6.996228218078613,
      "logps/rejected": -38.674766540527344,
      "loss": 0.8267521858215332,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5320640206336975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10675782710313797,
      "rewards/margins": 2.87463116645813,
      "rewards/rejected": -2.7678730487823486,
      "step": 274,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.46985957605851886,
      "grad_norm": 7.991383075714111,
      "learning_rate": 0.00018191970948552464,
      "logits/chosen": -0.8153300285339355,
      "logits/rejected": -0.8158446550369263,
      "logps/chosen": -8.539220809936523,
      "logps/rejected": -34.5740966796875,
      "loss": 0.5116520524024963,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2977459728717804,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13776344060897827,
      "rewards/margins": 2.662919282913208,
      "rewards/rejected": -2.525156021118164,
      "step": 275,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4715681563350953,
      "grad_norm": 4.298219203948975,
      "learning_rate": 0.00018175720924667056,
      "logits/chosen": -0.853663444519043,
      "logits/rejected": -0.8555605411529541,
      "logps/chosen": -7.882385730743408,
      "logps/rejected": -27.5541934967041,
      "loss": 0.5915999412536621,
      "memory(GiB)": 46.38,
      "nll_loss": 0.284732460975647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12615913152694702,
      "rewards/margins": 1.954941749572754,
      "rewards/rejected": -1.8287824392318726,
      "step": 276,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.47327673661167174,
      "grad_norm": 3.9946365356445312,
      "learning_rate": 0.00018159405527934996,
      "logits/chosen": -0.8487964868545532,
      "logits/rejected": -0.8562285304069519,
      "logps/chosen": -7.136427879333496,
      "logps/rejected": -37.46531677246094,
      "loss": 0.5510011315345764,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2794538140296936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.141952246427536,
      "rewards/margins": 2.4826676845550537,
      "rewards/rejected": -2.3407156467437744,
      "step": 277,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4749853168882482,
      "grad_norm": 1.9724056720733643,
      "learning_rate": 0.0001814302488881376,
      "logits/chosen": -0.8201677799224854,
      "logits/rejected": -0.8276962041854858,
      "logps/chosen": -4.926413536071777,
      "logps/rejected": -38.361087799072266,
      "loss": 0.45584461092948914,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2575777769088745,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.190385103225708,
      "rewards/margins": 2.507413387298584,
      "rewards/rejected": -2.317028522491455,
      "step": 278,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4766938971648246,
      "grad_norm": 8.133130073547363,
      "learning_rate": 0.00018126579138282503,
      "logits/chosen": -0.8125935792922974,
      "logits/rejected": -0.8118517398834229,
      "logps/chosen": -9.689276695251465,
      "logps/rejected": -29.489994049072266,
      "loss": 0.7013036608695984,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3926777243614197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06979987025260925,
      "rewards/margins": 2.248946189880371,
      "rewards/rejected": -2.1791462898254395,
      "step": 279,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.478402477441401,
      "grad_norm": 5.878256797790527,
      "learning_rate": 0.00018110068407841009,
      "logits/chosen": -0.8344768285751343,
      "logits/rejected": -0.8355271816253662,
      "logps/chosen": -7.216890335083008,
      "logps/rejected": -31.038660049438477,
      "loss": 0.6482636332511902,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3379790782928467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15157388150691986,
      "rewards/margins": 2.1786890029907227,
      "rewards/rejected": -2.0271151065826416,
      "step": 280,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.48011105771797746,
      "grad_norm": 3.594700336456299,
      "learning_rate": 0.00018093492829508636,
      "logits/chosen": -0.809922456741333,
      "logits/rejected": -0.8155569434165955,
      "logps/chosen": -8.301997184753418,
      "logps/rejected": -35.9427604675293,
      "loss": 0.7471737265586853,
      "memory(GiB)": 46.38,
      "nll_loss": 0.412964791059494,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.21606266498565674,
      "rewards/margins": 2.184842824935913,
      "rewards/rejected": -1.9687800407409668,
      "step": 281,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4818196379945539,
      "grad_norm": 4.715336799621582,
      "learning_rate": 0.00018076852535823268,
      "logits/chosen": -0.8024042844772339,
      "logits/rejected": -0.8047822713851929,
      "logps/chosen": -9.50023365020752,
      "logps/rejected": -26.15119743347168,
      "loss": 0.887703001499176,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4939606785774231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09452123939990997,
      "rewards/margins": 1.60733962059021,
      "rewards/rejected": -1.512818455696106,
      "step": 282,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.48352821827113035,
      "grad_norm": 2.6357319355010986,
      "learning_rate": 0.0001806014765984025,
      "logits/chosen": -0.7931514978408813,
      "logits/rejected": -0.7975669503211975,
      "logps/chosen": -4.370926856994629,
      "logps/rejected": -35.80842590332031,
      "loss": 0.4094904065132141,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17323070764541626,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25759178400039673,
      "rewards/margins": 2.2846627235412598,
      "rewards/rejected": -2.027071237564087,
      "step": 283,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.48523679854770674,
      "grad_norm": 4.521765232086182,
      "learning_rate": 0.00018043378335131326,
      "logits/chosen": -0.8225452899932861,
      "logits/rejected": -0.8211522698402405,
      "logps/chosen": -7.510035037994385,
      "logps/rejected": -36.8507194519043,
      "loss": 0.5709434747695923,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35384440422058105,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24606043100357056,
      "rewards/margins": 2.5631608963012695,
      "rewards/rejected": -2.3171002864837646,
      "step": 284,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4869453788242832,
      "grad_norm": 2.36458683013916,
      "learning_rate": 0.00018026544695783566,
      "logits/chosen": -0.8055735230445862,
      "logits/rejected": -0.8072599768638611,
      "logps/chosen": -8.788959503173828,
      "logps/rejected": -40.345008850097656,
      "loss": 0.5267823338508606,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34875571727752686,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.25422000885009766,
      "rewards/margins": 2.772775650024414,
      "rewards/rejected": -2.5185558795928955,
      "step": 285,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4886539591008596,
      "grad_norm": 10.764819145202637,
      "learning_rate": 0.00018009646876398307,
      "logits/chosen": -0.8059338331222534,
      "logits/rejected": -0.8106356263160706,
      "logps/chosen": -4.930308818817139,
      "logps/rejected": -31.040836334228516,
      "loss": 0.5658331513404846,
      "memory(GiB)": 46.38,
      "nll_loss": 0.28465500473976135,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1854298710823059,
      "rewards/margins": 1.930139422416687,
      "rewards/rejected": -1.7447094917297363,
      "step": 286,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.49036253937743607,
      "grad_norm": 3.613628387451172,
      "learning_rate": 0.0001799268501209006,
      "logits/chosen": -0.7612501382827759,
      "logits/rejected": -0.7627686858177185,
      "logps/chosen": -10.456869125366211,
      "logps/rejected": -28.227718353271484,
      "loss": 0.8326478004455566,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4999573230743408,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05842627212405205,
      "rewards/margins": 1.9171324968338013,
      "rewards/rejected": -1.8587062358856201,
      "step": 287,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.4920711196540125,
      "grad_norm": 4.662235736846924,
      "learning_rate": 0.0001797565923848544,
      "logits/chosen": -0.8021095991134644,
      "logits/rejected": -0.8046351671218872,
      "logps/chosen": -6.780048370361328,
      "logps/rejected": -31.340526580810547,
      "loss": 0.5166000723838806,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29529324173927307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24387100338935852,
      "rewards/margins": 2.386127233505249,
      "rewards/rejected": -2.142256259918213,
      "step": 288,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.4937796999305889,
      "grad_norm": 3.747556447982788,
      "learning_rate": 0.00017958569691722077,
      "logits/chosen": -0.7904728055000305,
      "logits/rejected": -0.7943806052207947,
      "logps/chosen": -4.052952766418457,
      "logps/rejected": -39.81676483154297,
      "loss": 0.444693922996521,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26461470127105713,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24267469346523285,
      "rewards/margins": 3.1456563472747803,
      "rewards/rejected": -2.9029815196990967,
      "step": 289,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.49548828020716534,
      "grad_norm": 3.8240482807159424,
      "learning_rate": 0.00017941416508447536,
      "logits/chosen": -0.8258907794952393,
      "logits/rejected": -0.8271006345748901,
      "logps/chosen": -12.335704803466797,
      "logps/rejected": -35.24673843383789,
      "loss": 0.8288266062736511,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5496087670326233,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.26559069752693176,
      "rewards/margins": 2.4179635047912598,
      "rewards/rejected": -2.1523728370666504,
      "step": 290,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.4971968604837418,
      "grad_norm": 5.618929386138916,
      "learning_rate": 0.0001792419982581821,
      "logits/chosen": -0.8527386784553528,
      "logits/rejected": -0.8554395437240601,
      "logps/chosen": -4.423219203948975,
      "logps/rejected": -37.118656158447266,
      "loss": 0.4001955986022949,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2208787500858307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2664373517036438,
      "rewards/margins": 2.81235933303833,
      "rewards/rejected": -2.545922040939331,
      "step": 291,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.49890544076031823,
      "grad_norm": 4.9184112548828125,
      "learning_rate": 0.00017906919781498235,
      "logits/chosen": -0.8180443048477173,
      "logits/rejected": -0.8243182897567749,
      "logps/chosen": -7.195798873901367,
      "logps/rejected": -46.805763244628906,
      "loss": 0.50282222032547,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24749858677387238,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17263692617416382,
      "rewards/margins": 3.1018624305725098,
      "rewards/rejected": -2.929225444793701,
      "step": 292,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.5006140210368947,
      "grad_norm": 6.550334930419922,
      "learning_rate": 0.00017889576513658385,
      "logits/chosen": -0.8828959465026855,
      "logits/rejected": -0.8903377652168274,
      "logps/chosen": -5.666334629058838,
      "logps/rejected": -41.20561218261719,
      "loss": 0.5666110515594482,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27025288343429565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14388155937194824,
      "rewards/margins": 2.9238851070404053,
      "rewards/rejected": -2.780003547668457,
      "step": 293,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.5023226013134711,
      "grad_norm": 5.214411735534668,
      "learning_rate": 0.0001787217016097497,
      "logits/chosen": -0.8843887448310852,
      "logits/rejected": -0.8880648612976074,
      "logps/chosen": -8.593646049499512,
      "logps/rejected": -31.845436096191406,
      "loss": 0.7616337537765503,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4269709587097168,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03095342591404915,
      "rewards/margins": 2.152078628540039,
      "rewards/rejected": -2.1211252212524414,
      "step": 294,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5040311815900476,
      "grad_norm": 3.419295310974121,
      "learning_rate": 0.00017854700862628717,
      "logits/chosen": -0.8697606325149536,
      "logits/rejected": -0.8711093068122864,
      "logps/chosen": -12.32712459564209,
      "logps/rejected": -35.99604415893555,
      "loss": 0.7168628573417664,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4990694522857666,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.00020820647478103638,
      "rewards/margins": 2.4814581871032715,
      "rewards/rejected": -2.481666088104248,
      "step": 295,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.505739761866624,
      "grad_norm": 5.47044038772583,
      "learning_rate": 0.00017837168758303673,
      "logits/chosen": -0.9019865989685059,
      "logits/rejected": -0.9105886220932007,
      "logps/chosen": -6.227141380310059,
      "logps/rejected": -53.5411376953125,
      "loss": 0.4358929395675659,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2536502182483673,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18560552597045898,
      "rewards/margins": 4.006980895996094,
      "rewards/rejected": -3.8213753700256348,
      "step": 296,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5074483421432003,
      "grad_norm": 2.2059149742126465,
      "learning_rate": 0.00017819573988186077,
      "logits/chosen": -0.8752774000167847,
      "logits/rejected": -0.8806784749031067,
      "logps/chosen": -6.102680206298828,
      "logps/rejected": -40.7698860168457,
      "loss": 0.5294958353042603,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32509148120880127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20541568100452423,
      "rewards/margins": 2.9198334217071533,
      "rewards/rejected": -2.7144179344177246,
      "step": 297,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5091569224197768,
      "grad_norm": 1.8315078020095825,
      "learning_rate": 0.00017801916692963246,
      "logits/chosen": -0.888721764087677,
      "logits/rejected": -0.8927589654922485,
      "logps/chosen": -5.959261417388916,
      "logps/rejected": -39.12718963623047,
      "loss": 0.41302210092544556,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25391918420791626,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.36076462268829346,
      "rewards/margins": 2.8598427772521973,
      "rewards/rejected": -2.4990782737731934,
      "step": 298,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.5108655026963532,
      "grad_norm": 2.749384641647339,
      "learning_rate": 0.00017784197013822435,
      "logits/chosen": -0.8662399053573608,
      "logits/rejected": -0.8707250356674194,
      "logps/chosen": -9.147662162780762,
      "logps/rejected": -45.20575714111328,
      "loss": 0.6315680146217346,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45470985770225525,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07495459169149399,
      "rewards/margins": 3.035630226135254,
      "rewards/rejected": -2.9606757164001465,
      "step": 299,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.5125740829729297,
      "grad_norm": 8.072540283203125,
      "learning_rate": 0.0001776641509244973,
      "logits/chosen": -0.8409525156021118,
      "logits/rejected": -0.8496344685554504,
      "logps/chosen": -6.282418251037598,
      "logps/rejected": -41.63984680175781,
      "loss": 0.871893584728241,
      "memory(GiB)": 46.38,
      "nll_loss": 0.603092610836029,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04116540029644966,
      "rewards/margins": 2.8330185413360596,
      "rewards/rejected": -2.7918529510498047,
      "step": 300,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5125740829729297,
      "eval_logits/chosen": -0.8405543565750122,
      "eval_logits/rejected": -0.8445731401443481,
      "eval_logps/chosen": -8.006686210632324,
      "eval_logps/rejected": -40.67301559448242,
      "eval_loss": 0.7707608342170715,
      "eval_nll_loss": 0.5364512801170349,
      "eval_rewards/accuracies": 0.8994709253311157,
      "eval_rewards/chosen": 0.06882528215646744,
      "eval_rewards/margins": 2.7935357093811035,
      "eval_rewards/rejected": -2.724710702896118,
      "eval_runtime": 442.8747,
      "eval_samples_per_second": 0.427,
      "eval_steps_per_second": 0.427,
      "step": 300
    },
    {
      "epoch": 0.5142826632495061,
      "grad_norm": 11.069442749023438,
      "learning_rate": 0.000177485710710289,
      "logits/chosen": -0.8602814674377441,
      "logits/rejected": -0.8637278079986572,
      "logps/chosen": -7.823677062988281,
      "logps/rejected": -39.174312591552734,
      "loss": 0.45352113246917725,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32754451036453247,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15461236238479614,
      "rewards/margins": 3.001373291015625,
      "rewards/rejected": -2.8467607498168945,
      "step": 301,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 0.5159912435260825,
      "grad_norm": 4.37824821472168,
      "learning_rate": 0.0001773066509224026,
      "logits/chosen": -0.7942736744880676,
      "logits/rejected": -0.7966884970664978,
      "logps/chosen": -7.395168304443359,
      "logps/rejected": -37.037654876708984,
      "loss": 0.6269878149032593,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4148014783859253,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27507781982421875,
      "rewards/margins": 2.9940710067749023,
      "rewards/rejected": -2.7189929485321045,
      "step": 302,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 0.517699823802659,
      "grad_norm": 1.2747763395309448,
      "learning_rate": 0.0001771269729925955,
      "logits/chosen": -0.8845440745353699,
      "logits/rejected": -0.8931821584701538,
      "logps/chosen": -4.624814510345459,
      "logps/rejected": -54.01844787597656,
      "loss": 0.3171394467353821,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2531016170978546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3055804371833801,
      "rewards/margins": 4.609305381774902,
      "rewards/rejected": -4.303724765777588,
      "step": 303,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5194084040792354,
      "grad_norm": 19.4792423248291,
      "learning_rate": 0.00017694667835756758,
      "logits/chosen": -0.8769544363021851,
      "logits/rejected": -0.8785020112991333,
      "logps/chosen": -13.249614715576172,
      "logps/rejected": -43.8223991394043,
      "loss": 1.2327057123184204,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8601534962654114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30870580673217773,
      "rewards/margins": 2.7113828659057617,
      "rewards/rejected": -3.0200884342193604,
      "step": 304,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5211169843558119,
      "grad_norm": 9.179790496826172,
      "learning_rate": 0.0001767657684589499,
      "logits/chosen": -0.8855112195014954,
      "logits/rejected": -0.8912838101387024,
      "logps/chosen": -7.211820602416992,
      "logps/rejected": -50.378562927246094,
      "loss": 0.7304961085319519,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42998042702674866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09179924428462982,
      "rewards/margins": 4.0141072273254395,
      "rewards/rejected": -3.9223079681396484,
      "step": 305,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5228255646323883,
      "grad_norm": 1.2901352643966675,
      "learning_rate": 0.00017658424474329326,
      "logits/chosen": -0.8675682544708252,
      "logits/rejected": -0.8787212371826172,
      "logps/chosen": -6.3633317947387695,
      "logps/rejected": -68.60183715820312,
      "loss": 0.31616559624671936,
      "memory(GiB)": 46.38,
      "nll_loss": 0.265287309885025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21000903844833374,
      "rewards/margins": 5.580310344696045,
      "rewards/rejected": -5.370301723480225,
      "step": 306,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5245341449089647,
      "grad_norm": 4.6658782958984375,
      "learning_rate": 0.00017640210866205639,
      "logits/chosen": -0.906684935092926,
      "logits/rejected": -0.9144970774650574,
      "logps/chosen": -9.675024032592773,
      "logps/rejected": -51.09449768066406,
      "loss": 0.6057628989219666,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4204673767089844,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10616739839315414,
      "rewards/margins": 3.975714921951294,
      "rewards/rejected": -4.081882476806641,
      "step": 307,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5262427251855412,
      "grad_norm": 5.465089321136475,
      "learning_rate": 0.00017621936167159458,
      "logits/chosen": -0.9267672300338745,
      "logits/rejected": -0.9362570643424988,
      "logps/chosen": -7.424238681793213,
      "logps/rejected": -65.96898651123047,
      "loss": 0.47528597712516785,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3133973479270935,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.09580647945404053,
      "rewards/margins": 5.268470287322998,
      "rewards/rejected": -5.364277362823486,
      "step": 308,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5279513054621175,
      "grad_norm": 5.624989986419678,
      "learning_rate": 0.00017603600523314787,
      "logits/chosen": -0.898685097694397,
      "logits/rejected": -0.8996263146400452,
      "logps/chosen": -12.03923225402832,
      "logps/rejected": -41.64332962036133,
      "loss": 0.7952063083648682,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4172392785549164,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.023442525416612625,
      "rewards/margins": 3.3135008811950684,
      "rewards/rejected": -3.2900586128234863,
      "step": 309,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.529659885738694,
      "grad_norm": 3.880650043487549,
      "learning_rate": 0.0001758520408128295,
      "logits/chosen": -0.8696607947349548,
      "logits/rejected": -0.8726595640182495,
      "logps/chosen": -7.42938232421875,
      "logps/rejected": -56.99799346923828,
      "loss": 0.5451213717460632,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3175729513168335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16648581624031067,
      "rewards/margins": 4.477063179016113,
      "rewards/rejected": -4.310576915740967,
      "step": 310,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5313684660152704,
      "grad_norm": 2.883660316467285,
      "learning_rate": 0.00017566746988161402,
      "logits/chosen": -0.8406087756156921,
      "logits/rejected": -0.8471619486808777,
      "logps/chosen": -9.634123802185059,
      "logps/rejected": -38.94046401977539,
      "loss": 0.6529473662376404,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38416606187820435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13221141695976257,
      "rewards/margins": 3.1708903312683105,
      "rewards/rejected": -3.0386788845062256,
      "step": 311,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5330770462918468,
      "grad_norm": 11.171285629272461,
      "learning_rate": 0.00017548229391532572,
      "logits/chosen": -0.8072686791419983,
      "logits/rejected": -0.8102389574050903,
      "logps/chosen": -6.779449462890625,
      "logps/rejected": -49.09239959716797,
      "loss": 0.5393321514129639,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3658963441848755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2095126211643219,
      "rewards/margins": 3.693910598754883,
      "rewards/rejected": -3.4843978881835938,
      "step": 312,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5347856265684233,
      "grad_norm": 1.954391360282898,
      "learning_rate": 0.00017529651439462664,
      "logits/chosen": -0.7809416055679321,
      "logits/rejected": -0.7857435345649719,
      "logps/chosen": -6.740390777587891,
      "logps/rejected": -48.86231994628906,
      "loss": 0.36391139030456543,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2839314639568329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31254059076309204,
      "rewards/margins": 3.944000482559204,
      "rewards/rejected": -3.6314597129821777,
      "step": 313,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5364942068449997,
      "grad_norm": 6.087454319000244,
      "learning_rate": 0.00017511013280500494,
      "logits/chosen": -0.7839592695236206,
      "logits/rejected": -0.7883790731430054,
      "logps/chosen": -5.397429943084717,
      "logps/rejected": -38.226078033447266,
      "loss": 0.47547078132629395,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2724950909614563,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16703476011753082,
      "rewards/margins": 2.7412147521972656,
      "rewards/rejected": -2.5741801261901855,
      "step": 314,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5382027871215762,
      "grad_norm": 7.710384845733643,
      "learning_rate": 0.0001749231506367628,
      "logits/chosen": -0.7206589579582214,
      "logits/rejected": -0.7282582521438599,
      "logps/chosen": -8.630600929260254,
      "logps/rejected": -40.66322708129883,
      "loss": 0.7258225679397583,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4553995132446289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14358378946781158,
      "rewards/margins": 2.626176595687866,
      "rewards/rejected": -2.482593059539795,
      "step": 315,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5399113673981526,
      "grad_norm": 1.8784159421920776,
      "learning_rate": 0.0001747355693850047,
      "logits/chosen": -0.7431631684303284,
      "logits/rejected": -0.748499870300293,
      "logps/chosen": -11.859859466552734,
      "logps/rejected": -43.0447883605957,
      "loss": 0.6120511293411255,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4628870189189911,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1693616509437561,
      "rewards/margins": 2.6937308311462402,
      "rewards/rejected": -2.524369239807129,
      "step": 316,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.541619947674729,
      "grad_norm": 2.4822375774383545,
      "learning_rate": 0.00017454739054962527,
      "logits/chosen": -0.7296395301818848,
      "logits/rejected": -0.7350925207138062,
      "logps/chosen": -7.4225616455078125,
      "logps/rejected": -44.191368103027344,
      "loss": 0.5170390605926514,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3234650194644928,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14995595812797546,
      "rewards/margins": 3.103259563446045,
      "rewards/rejected": -2.953303813934326,
      "step": 317,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5433285279513055,
      "grad_norm": 1.8158119916915894,
      "learning_rate": 0.00017435861563529746,
      "logits/chosen": -0.7801295518875122,
      "logits/rejected": -0.7813860177993774,
      "logps/chosen": -5.427134990692139,
      "logps/rejected": -36.239803314208984,
      "loss": 0.36503538489341736,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19436712563037872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22437262535095215,
      "rewards/margins": 2.704486131668091,
      "rewards/rejected": -2.4801137447357178,
      "step": 318,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5450371082278819,
      "grad_norm": 1.2651731967926025,
      "learning_rate": 0.00017416924615146055,
      "logits/chosen": -0.7636649012565613,
      "logits/rejected": -0.7674313187599182,
      "logps/chosen": -4.830596923828125,
      "logps/rejected": -49.42289733886719,
      "loss": 0.2704404294490814,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17981436848640442,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2880362272262573,
      "rewards/margins": 3.9220433235168457,
      "rewards/rejected": -3.634007215499878,
      "step": 319,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5467456885044584,
      "grad_norm": 2.813702344894409,
      "learning_rate": 0.0001739792836123078,
      "logits/chosen": -0.763135552406311,
      "logits/rejected": -0.7641803622245789,
      "logps/chosen": -8.197504043579102,
      "logps/rejected": -46.546730041503906,
      "loss": 0.43457722663879395,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3029811978340149,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21454845368862152,
      "rewards/margins": 3.952043056488037,
      "rewards/rejected": -3.737494707107544,
      "step": 320,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5484542687810348,
      "grad_norm": 10.072509765625,
      "learning_rate": 0.00017378872953677463,
      "logits/chosen": -0.7955141663551331,
      "logits/rejected": -0.8014116883277893,
      "logps/chosen": -6.334041595458984,
      "logps/rejected": -47.14308166503906,
      "loss": 0.4273318648338318,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26563501358032227,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.30760833621025085,
      "rewards/margins": 3.725619316101074,
      "rewards/rejected": -3.418011426925659,
      "step": 321,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5501628490576111,
      "grad_norm": 11.081689834594727,
      "learning_rate": 0.00017359758544852635,
      "logits/chosen": -0.8159039616584778,
      "logits/rejected": -0.8194667100906372,
      "logps/chosen": -9.277294158935547,
      "logps/rejected": -51.40448760986328,
      "loss": 0.8425000309944153,
      "memory(GiB)": 46.38,
      "nll_loss": 0.717254102230072,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.03732386231422424,
      "rewards/margins": 4.070914268493652,
      "rewards/rejected": -4.033590316772461,
      "step": 322,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5518714293341876,
      "grad_norm": 2.0417747497558594,
      "learning_rate": 0.00017340585287594604,
      "logits/chosen": -0.8340936899185181,
      "logits/rejected": -0.8355550169944763,
      "logps/chosen": -8.740560531616211,
      "logps/rejected": -48.48387908935547,
      "loss": 0.5082376599311829,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3596939742565155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3152270019054413,
      "rewards/margins": 3.8847768306732178,
      "rewards/rejected": -3.569550037384033,
      "step": 323,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.553580009610764,
      "grad_norm": 11.896222114562988,
      "learning_rate": 0.00017321353335212215,
      "logits/chosen": -0.8562039732933044,
      "logits/rejected": -0.8574480414390564,
      "logps/chosen": -9.903059005737305,
      "logps/rejected": -48.68834686279297,
      "loss": 0.5927642583847046,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3851415812969208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1212349608540535,
      "rewards/margins": 3.81803560256958,
      "rewards/rejected": -3.696800708770752,
      "step": 324,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5552885898873405,
      "grad_norm": 3.4423718452453613,
      "learning_rate": 0.00017302062841483655,
      "logits/chosen": -0.8601418733596802,
      "logits/rejected": -0.8596406579017639,
      "logps/chosen": -9.926023483276367,
      "logps/rejected": -53.83884811401367,
      "loss": 0.45005056262016296,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34139543771743774,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20918819308280945,
      "rewards/margins": 4.317967891693115,
      "rewards/rejected": -4.108779430389404,
      "step": 325,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5569971701639169,
      "grad_norm": 1.2915583848953247,
      "learning_rate": 0.00017282713960655195,
      "logits/chosen": -0.9310311079025269,
      "logits/rejected": -0.9432608485221863,
      "logps/chosen": -3.527310848236084,
      "logps/rejected": -56.88557434082031,
      "loss": 0.257373183965683,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18794924020767212,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.31664133071899414,
      "rewards/margins": 4.63478422164917,
      "rewards/rejected": -4.318142890930176,
      "step": 326,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5587057504404933,
      "grad_norm": 12.496362686157227,
      "learning_rate": 0.00017263306847439968,
      "logits/chosen": -0.8998693227767944,
      "logits/rejected": -0.9072844386100769,
      "logps/chosen": -7.473057270050049,
      "logps/rejected": -62.43585968017578,
      "loss": 0.4956499934196472,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36785978078842163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2565673589706421,
      "rewards/margins": 4.963648796081543,
      "rewards/rejected": -4.707081317901611,
      "step": 327,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5604143307170698,
      "grad_norm": 4.157985210418701,
      "learning_rate": 0.0001724384165701674,
      "logits/chosen": -0.9274346828460693,
      "logits/rejected": -0.9325964450836182,
      "logps/chosen": -7.024556636810303,
      "logps/rejected": -62.318359375,
      "loss": 0.35737261176109314,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25133123993873596,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11644118279218674,
      "rewards/margins": 4.924075126647949,
      "rewards/rejected": -4.807634353637695,
      "step": 328,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5621229109936462,
      "grad_norm": 4.035698413848877,
      "learning_rate": 0.00017224318545028649,
      "logits/chosen": -0.9244407415390015,
      "logits/rejected": -0.9326272010803223,
      "logps/chosen": -6.016265392303467,
      "logps/rejected": -56.14576721191406,
      "loss": 0.3880583643913269,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3103344738483429,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07886901497840881,
      "rewards/margins": 4.746619701385498,
      "rewards/rejected": -4.667750358581543,
      "step": 329,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5638314912702227,
      "grad_norm": 9.743223190307617,
      "learning_rate": 0.0001720473766758198,
      "logits/chosen": -0.926537275314331,
      "logits/rejected": -0.9304601550102234,
      "logps/chosen": -13.121172904968262,
      "logps/rejected": -58.38237380981445,
      "loss": 0.7646811008453369,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5820285677909851,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09494726359844208,
      "rewards/margins": 4.842239856719971,
      "rewards/rejected": -4.9371867179870605,
      "step": 330,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5655400715467991,
      "grad_norm": 18.880859375,
      "learning_rate": 0.00017185099181244907,
      "logits/chosen": -0.9021615386009216,
      "logits/rejected": -0.9024704694747925,
      "logps/chosen": -9.637939453125,
      "logps/rejected": -58.98625183105469,
      "loss": 0.739081859588623,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6550078392028809,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.00911126658320427,
      "rewards/margins": 5.053913116455078,
      "rewards/rejected": -5.063024997711182,
      "step": 331,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5672486518233755,
      "grad_norm": 2.728888750076294,
      "learning_rate": 0.00017165403243046248,
      "logits/chosen": -0.9438092112541199,
      "logits/rejected": -0.9487511515617371,
      "logps/chosen": -10.97130012512207,
      "logps/rejected": -60.31357955932617,
      "loss": 0.48772644996643066,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4556671679019928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31157898902893066,
      "rewards/margins": 5.334751129150391,
      "rewards/rejected": -5.023171901702881,
      "step": 332,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.568957232099952,
      "grad_norm": 17.81595802307129,
      "learning_rate": 0.00017145650010474194,
      "logits/chosen": -0.9099326133728027,
      "logits/rejected": -0.913872241973877,
      "logps/chosen": -7.330003261566162,
      "logps/rejected": -56.975791931152344,
      "loss": 0.44156697392463684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3481760025024414,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13420407474040985,
      "rewards/margins": 4.564350128173828,
      "rewards/rejected": -4.430145740509033,
      "step": 333,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5706658123765284,
      "grad_norm": 4.562387466430664,
      "learning_rate": 0.00017125839641475072,
      "logits/chosen": -0.93560391664505,
      "logits/rejected": -0.9402634501457214,
      "logps/chosen": -8.35469913482666,
      "logps/rejected": -48.29425048828125,
      "loss": 0.8423429727554321,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5765202045440674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20604895055294037,
      "rewards/margins": 4.131405353546143,
      "rewards/rejected": -3.925356149673462,
      "step": 334,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5723743926531049,
      "grad_norm": 6.9514899253845215,
      "learning_rate": 0.00017105972294452056,
      "logits/chosen": -0.9457940459251404,
      "logits/rejected": -0.9509940147399902,
      "logps/chosen": -8.684478759765625,
      "logps/rejected": -42.20777893066406,
      "loss": 0.6009758114814758,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2808820307254791,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.007849723100662231,
      "rewards/margins": 3.273137092590332,
      "rewards/rejected": -3.265286922454834,
      "step": 335,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5740829729296812,
      "grad_norm": 41.98228454589844,
      "learning_rate": 0.0001708604812826393,
      "logits/chosen": -0.9295578598976135,
      "logits/rejected": -0.935192346572876,
      "logps/chosen": -9.62477970123291,
      "logps/rejected": -54.46697235107422,
      "loss": 0.598048746585846,
      "memory(GiB)": 46.38,
      "nll_loss": 0.43297702074050903,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07474887371063232,
      "rewards/margins": 4.1404008865356445,
      "rewards/rejected": -4.065651893615723,
      "step": 336,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5757915532062576,
      "grad_norm": 2.598214626312256,
      "learning_rate": 0.00017066067302223798,
      "logits/chosen": -0.8595199584960938,
      "logits/rejected": -0.8645521402359009,
      "logps/chosen": -5.353518962860107,
      "logps/rejected": -45.4082145690918,
      "loss": 0.3949076235294342,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3393931984901428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17312097549438477,
      "rewards/margins": 3.7592127323150635,
      "rewards/rejected": -3.5860915184020996,
      "step": 337,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5775001334828341,
      "grad_norm": 6.433867454528809,
      "learning_rate": 0.00017046029976097805,
      "logits/chosen": -0.8666653633117676,
      "logits/rejected": -0.8727781176567078,
      "logps/chosen": -7.716047286987305,
      "logps/rejected": -44.40462112426758,
      "loss": 0.7068074345588684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4196939468383789,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.018908580765128136,
      "rewards/margins": 3.310499429702759,
      "rewards/rejected": -3.291590929031372,
      "step": 338,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5792087137594105,
      "grad_norm": 4.851775169372559,
      "learning_rate": 0.00017025936310103882,
      "logits/chosen": -0.8393141627311707,
      "logits/rejected": -0.8419257402420044,
      "logps/chosen": -10.38674259185791,
      "logps/rejected": -41.55541229248047,
      "loss": 0.5500025749206543,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35587984323501587,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21891042590141296,
      "rewards/margins": 3.442535400390625,
      "rewards/rejected": -3.2236249446868896,
      "step": 339,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.580917294035987,
      "grad_norm": 5.706538200378418,
      "learning_rate": 0.0001700578646491045,
      "logits/chosen": -0.849159300327301,
      "logits/rejected": -0.8539773225784302,
      "logps/chosen": -5.776857852935791,
      "logps/rejected": -41.82091522216797,
      "loss": 0.4652997851371765,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1927475780248642,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1367129385471344,
      "rewards/margins": 2.951298952102661,
      "rewards/rejected": -2.8145859241485596,
      "step": 340,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5826258743125634,
      "grad_norm": 15.33443832397461,
      "learning_rate": 0.00016985580601635124,
      "logits/chosen": -0.8022753000259399,
      "logits/rejected": -0.809974193572998,
      "logps/chosen": -10.902517318725586,
      "logps/rejected": -38.2662467956543,
      "loss": 0.7963184118270874,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5153663158416748,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.12418508529663086,
      "rewards/margins": 2.705582618713379,
      "rewards/rejected": -2.58139705657959,
      "step": 341,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5843344545891398,
      "grad_norm": 2.2032649517059326,
      "learning_rate": 0.00016965318881843457,
      "logits/chosen": -0.7673312425613403,
      "logits/rejected": -0.7743449211120605,
      "logps/chosen": -8.663345336914062,
      "logps/rejected": -45.02417755126953,
      "loss": 0.5525592565536499,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42993825674057007,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.231624573469162,
      "rewards/margins": 3.4039034843444824,
      "rewards/rejected": -3.172279119491577,
      "step": 342,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.5860430348657163,
      "grad_norm": 2.8073010444641113,
      "learning_rate": 0.0001694500146754762,
      "logits/chosen": -0.8122238516807556,
      "logits/rejected": -0.8119744658470154,
      "logps/chosen": -8.633512496948242,
      "logps/rejected": -35.36651611328125,
      "loss": 0.5415773391723633,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31399714946746826,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15843312442302704,
      "rewards/margins": 2.734858989715576,
      "rewards/rejected": -2.576425790786743,
      "step": 343,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5877516151422927,
      "grad_norm": 2.3773810863494873,
      "learning_rate": 0.0001692462852120511,
      "logits/chosen": -0.7741044759750366,
      "logits/rejected": -0.7800744771957397,
      "logps/chosen": -9.291975021362305,
      "logps/rejected": -40.77639389038086,
      "loss": 0.5208418965339661,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3624192774295807,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2715660035610199,
      "rewards/margins": 2.76344633102417,
      "rewards/rejected": -2.491880416870117,
      "step": 344,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5894601954188692,
      "grad_norm": 5.773314952850342,
      "learning_rate": 0.0001690420020571747,
      "logits/chosen": -0.7945303916931152,
      "logits/rejected": -0.8053185343742371,
      "logps/chosen": -7.3475542068481445,
      "logps/rejected": -50.37881851196289,
      "loss": 0.42357194423675537,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32000547647476196,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3487846553325653,
      "rewards/margins": 3.4395201206207275,
      "rewards/rejected": -3.090735673904419,
      "step": 345,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5911687756954456,
      "grad_norm": 6.197271347045898,
      "learning_rate": 0.0001688371668442896,
      "logits/chosen": -0.805290937423706,
      "logits/rejected": -0.8079847097396851,
      "logps/chosen": -7.421670436859131,
      "logps/rejected": -37.118797302246094,
      "loss": 0.5294437408447266,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3508721590042114,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17643693089485168,
      "rewards/margins": 2.624368667602539,
      "rewards/rejected": -2.4479315280914307,
      "step": 346,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.592877355972022,
      "grad_norm": 3.835904836654663,
      "learning_rate": 0.00016863178121125276,
      "logits/chosen": -0.8487049341201782,
      "logits/rejected": -0.8516339063644409,
      "logps/chosen": -8.74366283416748,
      "logps/rejected": -34.4682502746582,
      "loss": 0.5114731788635254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3230685293674469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0005252091214060783,
      "rewards/margins": 2.4223427772521973,
      "rewards/rejected": -2.4218177795410156,
      "step": 347,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.5945859362485985,
      "grad_norm": 14.421294212341309,
      "learning_rate": 0.00016842584680032223,
      "logits/chosen": -0.8961611986160278,
      "logits/rejected": -0.9048024415969849,
      "logps/chosen": -5.175715923309326,
      "logps/rejected": -48.69798278808594,
      "loss": 0.7653312683105469,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5099776387214661,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08455780148506165,
      "rewards/margins": 3.585397720336914,
      "rewards/rejected": -3.5008397102355957,
      "step": 348,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5962945165251748,
      "grad_norm": 3.0503435134887695,
      "learning_rate": 0.000168219365258144,
      "logits/chosen": -0.9129156470298767,
      "logits/rejected": -0.9179389476776123,
      "logps/chosen": -10.671258926391602,
      "logps/rejected": -43.36078643798828,
      "loss": 0.37005069851875305,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25679123401641846,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24289637804031372,
      "rewards/margins": 3.3170793056488037,
      "rewards/rejected": -3.0741827487945557,
      "step": 349,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5980030968017513,
      "grad_norm": 21.91331672668457,
      "learning_rate": 0.00016801233823573908,
      "logits/chosen": -0.8929505944252014,
      "logits/rejected": -0.8917251229286194,
      "logps/chosen": -14.047558784484863,
      "logps/rejected": -31.77324676513672,
      "loss": 0.976524829864502,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6302264928817749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11739279329776764,
      "rewards/margins": 2.250826120376587,
      "rewards/rejected": -2.1334333419799805,
      "step": 350,
      "train_speed(iter/s)": 0.005449
    }
  ],
  "logging_steps": 1,
  "max_steps": 1170,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 2.5475296526336e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}