{
  "best_global_step": 600,
  "best_metric": 0.3543238,
  "best_model_checkpoint": "/data2/kdd/crag/cjz/output/task3_dpo_gt6_task2_gt60.5+task3gt5/v1-20250609-010021/checkpoint-600",
  "epoch": 1.1093491377008917,
  "eval_steps": 300,
  "global_step": 650,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0017085802765764322,
      "grad_norm": 9.203307151794434,
      "learning_rate": 3.3898305084745763e-06,
      "logits/chosen": -0.6409764885902405,
      "logits/rejected": -0.6413162350654602,
      "logps/chosen": -8.591513633728027,
      "logps/rejected": -9.922354698181152,
      "loss": 1.2514135837554932,
      "memory(GiB)": 45.3,
      "nll_loss": 0.5582665801048279,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.005423
    },
    {
      "epoch": 0.0034171605531528645,
      "grad_norm": 4.572653293609619,
      "learning_rate": 6.779661016949153e-06,
      "logits/chosen": -0.6253237724304199,
      "logits/rejected": -0.6232100129127502,
      "logps/chosen": -9.215431213378906,
      "logps/rejected": -6.302733898162842,
      "loss": 1.1804862022399902,
      "memory(GiB)": 45.3,
      "nll_loss": 0.4873391091823578,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.005426
    },
    {
      "epoch": 0.005125740829729297,
      "grad_norm": 10.135663986206055,
      "learning_rate": 1.016949152542373e-05,
      "logits/chosen": -0.5830379724502563,
      "logits/rejected": -0.5847949981689453,
      "logps/chosen": -6.387916088104248,
      "logps/rejected": -12.526434898376465,
      "loss": 1.1388403177261353,
      "memory(GiB)": 46.08,
      "nll_loss": 0.44991743564605713,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0018886500038206577,
      "rewards/margins": 0.008566470816731453,
      "rewards/rejected": -0.006677820347249508,
      "step": 3,
      "train_speed(iter/s)": 0.005372
    },
    {
      "epoch": 0.006834321106305729,
      "grad_norm": 8.09815788269043,
      "learning_rate": 1.3559322033898305e-05,
      "logits/chosen": -0.5831664800643921,
      "logits/rejected": -0.5881690382957458,
      "logps/chosen": -9.77461051940918,
      "logps/rejected": -15.103912353515625,
      "loss": 1.2644962072372437,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5699671506881714,
      "rewards/accuracies": 0.4375,
      "rewards/chosen": -0.0013185496209189296,
      "rewards/margins": -0.0025646828580647707,
      "rewards/rejected": 0.0012461334699764848,
      "step": 4,
      "train_speed(iter/s)": 0.005433
    },
    {
      "epoch": 0.008542901382882162,
      "grad_norm": 10.546202659606934,
      "learning_rate": 1.694915254237288e-05,
      "logits/chosen": -0.6210501194000244,
      "logits/rejected": -0.6261105537414551,
      "logps/chosen": -9.743502616882324,
      "logps/rejected": -13.954730987548828,
      "loss": 1.2041635513305664,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5165743231773376,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.014782106503844261,
      "rewards/margins": 0.011779396794736385,
      "rewards/rejected": 0.0030027113389223814,
      "step": 5,
      "train_speed(iter/s)": 0.005434
    },
    {
      "epoch": 0.010251481659458593,
      "grad_norm": 5.953731060028076,
      "learning_rate": 2.033898305084746e-05,
      "logits/chosen": -0.6194843649864197,
      "logits/rejected": -0.6225060224533081,
      "logps/chosen": -8.993276596069336,
      "logps/rejected": -11.800742149353027,
      "loss": 1.2383989095687866,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5413038730621338,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.02151685394346714,
      "rewards/margins": -0.004275719169527292,
      "rewards/rejected": 0.025792576372623444,
      "step": 6,
      "train_speed(iter/s)": 0.005421
    },
    {
      "epoch": 0.011960061936035027,
      "grad_norm": 3.4447383880615234,
      "learning_rate": 2.3728813559322036e-05,
      "logits/chosen": -0.5979399681091309,
      "logits/rejected": -0.597865104675293,
      "logps/chosen": -11.88587760925293,
      "logps/rejected": -12.513296127319336,
      "loss": 1.1787316799163818,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4771195948123932,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": 0.001614744309335947,
      "rewards/margins": -0.006370083428919315,
      "rewards/rejected": 0.007984823547303677,
      "step": 7,
      "train_speed(iter/s)": 0.005421
    },
    {
      "epoch": 0.013668642212611458,
      "grad_norm": 4.3422627449035645,
      "learning_rate": 2.711864406779661e-05,
      "logits/chosen": -0.5695681571960449,
      "logits/rejected": -0.5736703276634216,
      "logps/chosen": -8.691235542297363,
      "logps/rejected": -17.510940551757812,
      "loss": 1.085353970527649,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4569101631641388,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.061018284410238266,
      "rewards/margins": 0.16366331279277802,
      "rewards/rejected": -0.10264502465724945,
      "step": 8,
      "train_speed(iter/s)": 0.005421
    },
    {
      "epoch": 0.015377222489187891,
      "grad_norm": 6.347766399383545,
      "learning_rate": 3.050847457627119e-05,
      "logits/chosen": -0.6150330305099487,
      "logits/rejected": -0.6163787841796875,
      "logps/chosen": -14.26973819732666,
      "logps/rejected": -12.656587600708008,
      "loss": 1.3125040531158447,
      "memory(GiB)": 46.08,
      "nll_loss": 0.6051884293556213,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.06490564346313477,
      "rewards/margins": 0.03234899416565895,
      "rewards/rejected": -0.09725463390350342,
      "step": 9,
      "train_speed(iter/s)": 0.005427
    },
    {
      "epoch": 0.017085802765764324,
      "grad_norm": 3.609342575073242,
      "learning_rate": 3.389830508474576e-05,
      "logits/chosen": -0.5823943018913269,
      "logits/rejected": -0.5893073678016663,
      "logps/chosen": -12.539135932922363,
      "logps/rejected": -19.231651306152344,
      "loss": 1.154615044593811,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5292682647705078,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.0102485166862607,
      "rewards/margins": 0.24539220333099365,
      "rewards/rejected": -0.2556407153606415,
      "step": 10,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 0.018794383042340754,
      "grad_norm": 4.803543567657471,
      "learning_rate": 3.728813559322034e-05,
      "logits/chosen": -0.6298958659172058,
      "logits/rejected": -0.6301961541175842,
      "logps/chosen": -10.286799430847168,
      "logps/rejected": -11.364641189575195,
      "loss": 1.143830418586731,
      "memory(GiB)": 46.08,
      "nll_loss": 0.43475160002708435,
      "rewards/accuracies": 0.46875,
      "rewards/chosen": -0.06166127696633339,
      "rewards/margins": 0.1280723363161087,
      "rewards/rejected": -0.189733624458313,
      "step": 11,
      "train_speed(iter/s)": 0.005459
    },
    {
      "epoch": 0.020502963318917187,
      "grad_norm": 6.843598365783691,
      "learning_rate": 4.067796610169492e-05,
      "logits/chosen": -0.6147834062576294,
      "logits/rejected": -0.6201154589653015,
      "logps/chosen": -11.104015350341797,
      "logps/rejected": -14.71792221069336,
      "loss": 1.2689580917358398,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4583723843097687,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.1446399688720703,
      "rewards/margins": -0.06440749764442444,
      "rewards/rejected": -0.08023246377706528,
      "step": 12,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.02221154359549362,
      "grad_norm": 4.361201286315918,
      "learning_rate": 4.4067796610169495e-05,
      "logits/chosen": -0.6593450307846069,
      "logits/rejected": -0.6623726487159729,
      "logps/chosen": -8.788830757141113,
      "logps/rejected": -12.09660530090332,
      "loss": 1.091460108757019,
      "memory(GiB)": 46.08,
      "nll_loss": 0.38157719373703003,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": -0.03397805988788605,
      "rewards/margins": 0.08494514226913452,
      "rewards/rejected": -0.11892320215702057,
      "step": 13,
      "train_speed(iter/s)": 0.005465
    },
    {
      "epoch": 0.023920123872070053,
      "grad_norm": 4.36707067489624,
      "learning_rate": 4.745762711864407e-05,
      "logits/chosen": -0.6003404259681702,
      "logits/rejected": -0.6018444895744324,
      "logps/chosen": -9.472915649414062,
      "logps/rejected": -12.008777618408203,
      "loss": 1.2156641483306885,
      "memory(GiB)": 46.08,
      "nll_loss": 0.42055219411849976,
      "rewards/accuracies": 0.40625,
      "rewards/chosen": -0.07689498364925385,
      "rewards/margins": -0.1264859437942505,
      "rewards/rejected": 0.049590952694416046,
      "step": 14,
      "train_speed(iter/s)": 0.00548
    },
    {
      "epoch": 0.025628704148646483,
      "grad_norm": 3.2864296436309814,
      "learning_rate": 5.0847457627118643e-05,
      "logits/chosen": -0.6285042762756348,
      "logits/rejected": -0.6272621154785156,
      "logps/chosen": -14.201019287109375,
      "logps/rejected": -10.998590469360352,
      "loss": 1.2311502695083618,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5152750015258789,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.06618006527423859,
      "rewards/margins": 0.029741406440734863,
      "rewards/rejected": 0.036438651382923126,
      "step": 15,
      "train_speed(iter/s)": 0.005474
    },
    {
      "epoch": 0.027337284425222916,
      "grad_norm": 2.4144492149353027,
      "learning_rate": 5.423728813559322e-05,
      "logits/chosen": -0.6098039746284485,
      "logits/rejected": -0.6130300760269165,
      "logps/chosen": -3.4820291996002197,
      "logps/rejected": -16.196063995361328,
      "loss": 0.8393359780311584,
      "memory(GiB)": 46.08,
      "nll_loss": 0.214947909116745,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0826798751950264,
      "rewards/margins": 0.17492221295833588,
      "rewards/rejected": -0.09224233776330948,
      "step": 16,
      "train_speed(iter/s)": 0.005477
    },
    {
      "epoch": 0.02904586470179935,
      "grad_norm": 2.3232851028442383,
      "learning_rate": 5.76271186440678e-05,
      "logits/chosen": -0.5909145474433899,
      "logits/rejected": -0.593756377696991,
      "logps/chosen": -7.929438591003418,
      "logps/rejected": -14.363272666931152,
      "loss": 0.9911059737205505,
      "memory(GiB)": 46.08,
      "nll_loss": 0.34508591890335083,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.10390710830688477,
      "rewards/margins": 0.13225752115249634,
      "rewards/rejected": -0.028350409120321274,
      "step": 17,
      "train_speed(iter/s)": 0.005482
    },
    {
      "epoch": 0.030754444978375782,
      "grad_norm": 2.344477891921997,
      "learning_rate": 6.101694915254238e-05,
      "logits/chosen": -0.5697246789932251,
      "logits/rejected": -0.5692474246025085,
      "logps/chosen": -6.287097930908203,
      "logps/rejected": -10.676097869873047,
      "loss": 0.9974570274353027,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3142527639865875,
      "rewards/accuracies": 0.5625,
      "rewards/chosen": 0.09520465135574341,
      "rewards/margins": 0.04853628948330879,
      "rewards/rejected": 0.046668365597724915,
      "step": 18,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.03246302525495221,
      "grad_norm": 2.361417531967163,
      "learning_rate": 6.440677966101695e-05,
      "logits/chosen": -0.5909584164619446,
      "logits/rejected": -0.596694827079773,
      "logps/chosen": -4.706894874572754,
      "logps/rejected": -12.18591022491455,
      "loss": 0.8993738889694214,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2403678447008133,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.1180216372013092,
      "rewards/margins": 0.09891890734434128,
      "rewards/rejected": 0.01910274103283882,
      "step": 19,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.03417160553152865,
      "grad_norm": 2.1051840782165527,
      "learning_rate": 6.779661016949152e-05,
      "logits/chosen": -0.5925375819206238,
      "logits/rejected": -0.5939764380455017,
      "logps/chosen": -7.690135478973389,
      "logps/rejected": -11.360199928283691,
      "loss": 0.9811086058616638,
      "memory(GiB)": 46.08,
      "nll_loss": 0.27671271562576294,
      "rewards/accuracies": 0.53125,
      "rewards/chosen": 0.15354080498218536,
      "rewards/margins": 0.02018781006336212,
      "rewards/rejected": 0.13335299491882324,
      "step": 20,
      "train_speed(iter/s)": 0.005507
    },
    {
      "epoch": 0.03588018580810508,
      "grad_norm": 2.453577995300293,
      "learning_rate": 7.11864406779661e-05,
      "logits/chosen": -0.589929461479187,
      "logits/rejected": -0.5913950800895691,
      "logps/chosen": -6.006174564361572,
      "logps/rejected": -12.356660842895508,
      "loss": 0.8762722015380859,
      "memory(GiB)": 46.08,
      "nll_loss": 0.26102662086486816,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.15477147698402405,
      "rewards/margins": 0.20186343789100647,
      "rewards/rejected": -0.047091953456401825,
      "step": 21,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.03758876608468151,
      "grad_norm": 2.334592819213867,
      "learning_rate": 7.457627118644068e-05,
      "logits/chosen": -0.6286447048187256,
      "logits/rejected": -0.6310911178588867,
      "logps/chosen": -9.828102111816406,
      "logps/rejected": -16.171802520751953,
      "loss": 0.9827708601951599,
      "memory(GiB)": 46.08,
      "nll_loss": 0.40016523003578186,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.22255462408065796,
      "rewards/margins": 0.2890108525753021,
      "rewards/rejected": -0.06645623594522476,
      "step": 22,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.039297346361257944,
      "grad_norm": 2.763946533203125,
      "learning_rate": 7.796610169491526e-05,
      "logits/chosen": -0.6423724889755249,
      "logits/rejected": -0.6475900411605835,
      "logps/chosen": -7.048077583312988,
      "logps/rejected": -17.518115997314453,
      "loss": 1.029776692390442,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4258265495300293,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13568007946014404,
      "rewards/margins": 0.24192118644714355,
      "rewards/rejected": -0.10624107718467712,
      "step": 23,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.041005926637834374,
      "grad_norm": 5.438773155212402,
      "learning_rate": 8.135593220338983e-05,
      "logits/chosen": -0.611075758934021,
      "logits/rejected": -0.6109172701835632,
      "logps/chosen": -8.579011917114258,
      "logps/rejected": -12.74751091003418,
      "loss": 1.2176915407180786,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5213994979858398,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": -0.010570855811238289,
      "rewards/margins": 0.06740333139896393,
      "rewards/rejected": -0.07797417044639587,
      "step": 24,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.0427145069144108,
      "grad_norm": 3.8305866718292236,
      "learning_rate": 8.474576271186441e-05,
      "logits/chosen": -0.6644467711448669,
      "logits/rejected": -0.6674079298973083,
      "logps/chosen": -6.975298881530762,
      "logps/rejected": -15.503819465637207,
      "loss": 0.922063410282135,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3326256275177002,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.06792093813419342,
      "rewards/margins": 0.324211061000824,
      "rewards/rejected": -0.25629013776779175,
      "step": 25,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.04442308719098724,
      "grad_norm": 4.446774959564209,
      "learning_rate": 8.813559322033899e-05,
      "logits/chosen": -0.6409133076667786,
      "logits/rejected": -0.6452510356903076,
      "logps/chosen": -5.22960090637207,
      "logps/rejected": -16.918365478515625,
      "loss": 0.7774365544319153,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2448062300682068,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2400483787059784,
      "rewards/margins": 0.39764925837516785,
      "rewards/rejected": -0.15760089457035065,
      "step": 26,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.04613166746756367,
      "grad_norm": 2.5869643688201904,
      "learning_rate": 9.152542372881357e-05,
      "logits/chosen": -0.6091906428337097,
      "logits/rejected": -0.6120930910110474,
      "logps/chosen": -7.03964376449585,
      "logps/rejected": -18.091930389404297,
      "loss": 0.8498358726501465,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3106813132762909,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1571023166179657,
      "rewards/margins": 0.4190843403339386,
      "rewards/rejected": -0.2619820237159729,
      "step": 27,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.047840247744140106,
      "grad_norm": 3.0630600452423096,
      "learning_rate": 9.491525423728815e-05,
      "logits/chosen": -0.6014366745948792,
      "logits/rejected": -0.6016579270362854,
      "logps/chosen": -9.281977653503418,
      "logps/rejected": -13.255182266235352,
      "loss": 0.9999822974205017,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3916056454181671,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.0838853195309639,
      "rewards/margins": 0.22357328236103058,
      "rewards/rejected": -0.13968798518180847,
      "step": 28,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.049548828020716536,
      "grad_norm": 4.956308364868164,
      "learning_rate": 9.830508474576272e-05,
      "logits/chosen": -0.6361557841300964,
      "logits/rejected": -0.6375819444656372,
      "logps/chosen": -5.407459735870361,
      "logps/rejected": -12.960749626159668,
      "loss": 0.9380594491958618,
      "memory(GiB)": 46.08,
      "nll_loss": 0.39800193905830383,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14713114500045776,
      "rewards/margins": 0.4169777035713196,
      "rewards/rejected": -0.2698465585708618,
      "step": 29,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.051257408297292965,
      "grad_norm": 6.679542541503906,
      "learning_rate": 0.00010169491525423729,
      "logits/chosen": -0.6229231357574463,
      "logits/rejected": -0.6250395178794861,
      "logps/chosen": -11.77493667602539,
      "logps/rejected": -18.192317962646484,
      "loss": 0.7899165749549866,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2795858085155487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09651458263397217,
      "rewards/margins": 0.49087047576904297,
      "rewards/rejected": -0.3943559229373932,
      "step": 30,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.0529659885738694,
      "grad_norm": 4.011873245239258,
      "learning_rate": 0.00010508474576271188,
      "logits/chosen": -0.6524882912635803,
      "logits/rejected": -0.6541539430618286,
      "logps/chosen": -7.8713202476501465,
      "logps/rejected": -11.641451835632324,
      "loss": 0.9657379984855652,
      "memory(GiB)": 46.08,
      "nll_loss": 0.40233710408210754,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.11578395962715149,
      "rewards/margins": 0.36746829748153687,
      "rewards/rejected": -0.25168436765670776,
      "step": 31,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.05467456885044583,
      "grad_norm": 4.144499778747559,
      "learning_rate": 0.00010847457627118644,
      "logits/chosen": -0.6458109617233276,
      "logits/rejected": -0.6519441604614258,
      "logps/chosen": -8.17459487915039,
      "logps/rejected": -19.17450714111328,
      "loss": 0.9243566989898682,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3649522066116333,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.17762523889541626,
      "rewards/margins": 0.41990581154823303,
      "rewards/rejected": -0.24228057265281677,
      "step": 32,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.05638314912702227,
      "grad_norm": 2.5846283435821533,
      "learning_rate": 0.00011186440677966102,
      "logits/chosen": -0.6422857046127319,
      "logits/rejected": -0.64764004945755,
      "logps/chosen": -7.8391828536987305,
      "logps/rejected": -16.905941009521484,
      "loss": 0.8121039867401123,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3524104952812195,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.21112650632858276,
      "rewards/margins": 0.6391255259513855,
      "rewards/rejected": -0.42799898982048035,
      "step": 33,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.0580917294035987,
      "grad_norm": 2.831099271774292,
      "learning_rate": 0.0001152542372881356,
      "logits/chosen": -0.6351624727249146,
      "logits/rejected": -0.6340872645378113,
      "logps/chosen": -8.873239517211914,
      "logps/rejected": -14.13478946685791,
      "loss": 0.9939954280853271,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5031149983406067,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.15203362703323364,
      "rewards/margins": 0.5661364793777466,
      "rewards/rejected": -0.41410285234451294,
      "step": 34,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.05980030968017513,
      "grad_norm": 6.754673004150391,
      "learning_rate": 0.00011864406779661017,
      "logits/chosen": -0.6350009441375732,
      "logits/rejected": -0.6345161199569702,
      "logps/chosen": -9.049372673034668,
      "logps/rejected": -13.872842788696289,
      "loss": 0.9677298069000244,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4823642373085022,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18822304904460907,
      "rewards/margins": 0.5690968632698059,
      "rewards/rejected": -0.38087382912635803,
      "step": 35,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.061508889956751564,
      "grad_norm": 5.09066915512085,
      "learning_rate": 0.00012203389830508477,
      "logits/chosen": -0.5940477252006531,
      "logits/rejected": -0.5995637774467468,
      "logps/chosen": -6.552436351776123,
      "logps/rejected": -22.02668571472168,
      "loss": 1.0198695659637451,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5385282635688782,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.09731997549533844,
      "rewards/margins": 0.7183221578598022,
      "rewards/rejected": -0.621002197265625,
      "step": 36,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.063217470233328,
      "grad_norm": 6.366957664489746,
      "learning_rate": 0.00012542372881355933,
      "logits/chosen": -0.5671308040618896,
      "logits/rejected": -0.5699396133422852,
      "logps/chosen": -8.690061569213867,
      "logps/rejected": -18.227741241455078,
      "loss": 0.9119342565536499,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4206831753253937,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.008592324331402779,
      "rewards/margins": 0.7837947010993958,
      "rewards/rejected": -0.7752023935317993,
      "step": 37,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.06492605050990442,
      "grad_norm": 8.224360466003418,
      "learning_rate": 0.0001288135593220339,
      "logits/chosen": -0.6006415486335754,
      "logits/rejected": -0.6039485931396484,
      "logps/chosen": -7.98246955871582,
      "logps/rejected": -19.214567184448242,
      "loss": 1.020494818687439,
      "memory(GiB)": 46.08,
      "nll_loss": 0.47989213466644287,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.017101695761084557,
      "rewards/margins": 0.7686665654182434,
      "rewards/rejected": -0.7857682704925537,
      "step": 38,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.06663463078648085,
      "grad_norm": 5.284773349761963,
      "learning_rate": 0.00013220338983050849,
      "logits/chosen": -0.5746678709983826,
      "logits/rejected": -0.5742544531822205,
      "logps/chosen": -10.300456047058105,
      "logps/rejected": -14.731122970581055,
      "loss": 1.1951102018356323,
      "memory(GiB)": 46.08,
      "nll_loss": 0.6024460196495056,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.005457160994410515,
      "rewards/margins": 0.43640822172164917,
      "rewards/rejected": -0.44186532497406006,
      "step": 39,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.0683432110630573,
      "grad_norm": 4.581506252288818,
      "learning_rate": 0.00013559322033898305,
      "logits/chosen": -0.5436846613883972,
      "logits/rejected": -0.5412294268608093,
      "logps/chosen": -11.306229591369629,
      "logps/rejected": -15.808780670166016,
      "loss": 0.9979494214057922,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4069577753543854,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.10538371652364731,
      "rewards/margins": 0.4707936942577362,
      "rewards/rejected": -0.5761774778366089,
      "step": 40,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.07005179133963373,
      "grad_norm": 4.280210018157959,
      "learning_rate": 0.00013898305084745764,
      "logits/chosen": -0.5919702649116516,
      "logits/rejected": -0.5944460034370422,
      "logps/chosen": -4.601211071014404,
      "logps/rejected": -18.39055061340332,
      "loss": 0.7289348244667053,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2620496451854706,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1577654778957367,
      "rewards/margins": 0.7140093445777893,
      "rewards/rejected": -0.5562438368797302,
      "step": 41,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.07176037161621016,
      "grad_norm": 2.8965229988098145,
      "learning_rate": 0.0001423728813559322,
      "logits/chosen": -0.5226963758468628,
      "logits/rejected": -0.5210604667663574,
      "logps/chosen": -8.042407035827637,
      "logps/rejected": -13.117356300354004,
      "loss": 0.8382841944694519,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3229953646659851,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.15337654948234558,
      "rewards/margins": 0.5258168578147888,
      "rewards/rejected": -0.3724403381347656,
      "step": 42,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.07346895189278659,
      "grad_norm": 4.7869791984558105,
      "learning_rate": 0.00014576271186440677,
      "logits/chosen": -0.5595600605010986,
      "logits/rejected": -0.5619646310806274,
      "logps/chosen": -5.741379261016846,
      "logps/rejected": -14.867729187011719,
      "loss": 0.9325981140136719,
      "memory(GiB)": 46.08,
      "nll_loss": 0.34299078583717346,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.06792251765727997,
      "rewards/margins": 0.3079211115837097,
      "rewards/rejected": -0.23999860882759094,
      "step": 43,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.07517753216936301,
      "grad_norm": 3.8402724266052246,
      "learning_rate": 0.00014915254237288136,
      "logits/chosen": -0.5551996231079102,
      "logits/rejected": -0.5583171248435974,
      "logps/chosen": -7.729850769042969,
      "logps/rejected": -14.535001754760742,
      "loss": 0.985005795955658,
      "memory(GiB)": 46.08,
      "nll_loss": 0.37224525213241577,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.0475621297955513,
      "rewards/margins": 0.2842516601085663,
      "rewards/rejected": -0.23668958246707916,
      "step": 44,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.07688611244593946,
      "grad_norm": 3.136073112487793,
      "learning_rate": 0.00015254237288135592,
      "logits/chosen": -0.5538796186447144,
      "logits/rejected": -0.5563740730285645,
      "logps/chosen": -8.61514663696289,
      "logps/rejected": -20.434301376342773,
      "loss": 0.8447250723838806,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3852572441101074,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17051486670970917,
      "rewards/margins": 0.695674479007721,
      "rewards/rejected": -0.5251596570014954,
      "step": 45,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.07859469272251589,
      "grad_norm": 2.950076103210449,
      "learning_rate": 0.00015593220338983051,
      "logits/chosen": -0.588573694229126,
      "logits/rejected": -0.5898380875587463,
      "logps/chosen": -8.49618911743164,
      "logps/rejected": -16.47633171081543,
      "loss": 0.8776077032089233,
      "memory(GiB)": 46.08,
      "nll_loss": 0.3434315621852875,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1189945787191391,
      "rewards/margins": 0.5393712520599365,
      "rewards/rejected": -0.42037665843963623,
      "step": 46,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.08030327299909232,
      "grad_norm": 2.918926954269409,
      "learning_rate": 0.00015932203389830508,
      "logits/chosen": -0.5735058188438416,
      "logits/rejected": -0.5742441415786743,
      "logps/chosen": -10.007829666137695,
      "logps/rejected": -15.225828170776367,
      "loss": 0.9683640003204346,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4518345892429352,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.29521822929382324,
      "rewards/margins": 0.5080306529998779,
      "rewards/rejected": -0.2128123939037323,
      "step": 47,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.08201185327566875,
      "grad_norm": 2.885613441467285,
      "learning_rate": 0.00016271186440677967,
      "logits/chosen": -0.5690568089485168,
      "logits/rejected": -0.570725679397583,
      "logps/chosen": -6.9214887619018555,
      "logps/rejected": -17.073692321777344,
      "loss": 0.8013411164283752,
      "memory(GiB)": 46.08,
      "nll_loss": 0.2747730612754822,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1803244799375534,
      "rewards/margins": 0.5454539060592651,
      "rewards/rejected": -0.36512941122055054,
      "step": 48,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.08372043355224518,
      "grad_norm": 2.8386495113372803,
      "learning_rate": 0.00016610169491525423,
      "logits/chosen": -0.5691187381744385,
      "logits/rejected": -0.5723721981048584,
      "logps/chosen": -6.647555828094482,
      "logps/rejected": -18.68927764892578,
      "loss": 0.731940746307373,
      "memory(GiB)": 46.08,
      "nll_loss": 0.31857162714004517,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15964265167713165,
      "rewards/margins": 0.8172457218170166,
      "rewards/rejected": -0.6576029062271118,
      "step": 49,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.0854290138288216,
      "grad_norm": 6.678228855133057,
      "learning_rate": 0.00016949152542372882,
      "logits/chosen": -0.569486141204834,
      "logits/rejected": -0.5724078416824341,
      "logps/chosen": -7.978659629821777,
      "logps/rejected": -18.0915584564209,
      "loss": 0.8816509246826172,
      "memory(GiB)": 46.08,
      "nll_loss": 0.41086509823799133,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16925573348999023,
      "rewards/margins": 0.6901466846466064,
      "rewards/rejected": -0.520891010761261,
      "step": 50,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.08713759410539805,
      "grad_norm": 3.4269354343414307,
      "learning_rate": 0.00017288135593220342,
      "logits/chosen": -0.6229482889175415,
      "logits/rejected": -0.624045193195343,
      "logps/chosen": -8.981767654418945,
      "logps/rejected": -18.87827491760254,
      "loss": 0.902818500995636,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4753139615058899,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12949834764003754,
      "rewards/margins": 0.91382896900177,
      "rewards/rejected": -0.7843306064605713,
      "step": 51,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.08884617438197448,
      "grad_norm": 3.730417013168335,
      "learning_rate": 0.00017627118644067798,
      "logits/chosen": -0.5907954573631287,
      "logits/rejected": -0.5928065776824951,
      "logps/chosen": -8.826777458190918,
      "logps/rejected": -25.743032455444336,
      "loss": 0.8241711854934692,
      "memory(GiB)": 46.08,
      "nll_loss": 0.4183700382709503,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15070247650146484,
      "rewards/margins": 1.0849279165267944,
      "rewards/rejected": -0.9342254400253296,
      "step": 52,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.09055475465855091,
      "grad_norm": 6.543994426727295,
      "learning_rate": 0.00017966101694915257,
      "logits/chosen": -0.5657305717468262,
      "logits/rejected": -0.5669429898262024,
      "logps/chosen": -11.976017951965332,
      "logps/rejected": -18.324581146240234,
      "loss": 1.1098811626434326,
      "memory(GiB)": 46.08,
      "nll_loss": 0.5012481808662415,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.04995343089103699,
      "rewards/margins": 0.5761421918869019,
      "rewards/rejected": -0.6260955929756165,
      "step": 53,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.09226333493512734,
      "grad_norm": 7.451931476593018,
      "learning_rate": 0.00018305084745762714,
      "logits/chosen": -0.5736872553825378,
      "logits/rejected": -0.5727863907814026,
      "logps/chosen": -9.956096649169922,
      "logps/rejected": -17.354209899902344,
      "loss": 1.0036059617996216,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35273733735084534,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": -0.07980930060148239,
      "rewards/margins": 0.5828754901885986,
      "rewards/rejected": -0.6626847982406616,
      "step": 54,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.09397191521170377,
      "grad_norm": 4.901066303253174,
      "learning_rate": 0.0001864406779661017,
      "logits/chosen": -0.5865222811698914,
      "logits/rejected": -0.5857770442962646,
      "logps/chosen": -6.749457359313965,
      "logps/rejected": -19.966379165649414,
      "loss": 0.836351215839386,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30449604988098145,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.11927387118339539,
      "rewards/margins": 0.7463572025299072,
      "rewards/rejected": -0.6270833611488342,
      "step": 55,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.09568049548828021,
      "grad_norm": 3.0312962532043457,
      "learning_rate": 0.0001898305084745763,
      "logits/chosen": -0.5644817352294922,
      "logits/rejected": -0.5671055912971497,
      "logps/chosen": -6.525354862213135,
      "logps/rejected": -18.556304931640625,
      "loss": 0.7299084663391113,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2566152811050415,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.051953963935375214,
      "rewards/margins": 0.8131421804428101,
      "rewards/rejected": -0.7611882090568542,
      "step": 56,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.09738907576485664,
      "grad_norm": 6.161406517028809,
      "learning_rate": 0.00019322033898305085,
      "logits/chosen": -0.5638913512229919,
      "logits/rejected": -0.5684786438941956,
      "logps/chosen": -8.483434677124023,
      "logps/rejected": -20.397701263427734,
      "loss": 0.988010585308075,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4928280711174011,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.17585664987564087,
      "rewards/margins": 0.6749651432037354,
      "rewards/rejected": -0.4991084933280945,
      "step": 57,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.09909765604143307,
      "grad_norm": 4.03005313873291,
      "learning_rate": 0.00019661016949152545,
      "logits/chosen": -0.5518775582313538,
      "logits/rejected": -0.552839994430542,
      "logps/chosen": -6.419961929321289,
      "logps/rejected": -20.10483169555664,
      "loss": 0.7270359396934509,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2987552881240845,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23015271127223969,
      "rewards/margins": 0.8427435159683228,
      "rewards/rejected": -0.6125907897949219,
      "step": 58,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.1008062363180095,
      "grad_norm": 2.9903881549835205,
      "learning_rate": 0.0002,
      "logits/chosen": -0.558972954750061,
      "logits/rejected": -0.559365451335907,
      "logps/chosen": -7.266337871551514,
      "logps/rejected": -19.700899124145508,
      "loss": 0.8035504221916199,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35604721307754517,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23076069355010986,
      "rewards/margins": 0.6581640243530273,
      "rewards/rejected": -0.4274032711982727,
      "step": 59,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.10251481659458593,
      "grad_norm": 2.308953046798706,
      "learning_rate": 0.00019999960020133237,
      "logits/chosen": -0.5721818804740906,
      "logits/rejected": -0.5736841559410095,
      "logps/chosen": -5.192386150360107,
      "logps/rejected": -16.873666763305664,
      "loss": 0.6971458196640015,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25499823689460754,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.14774081110954285,
      "rewards/margins": 0.7478816509246826,
      "rewards/rejected": -0.6001408696174622,
      "step": 60,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.10422339687116237,
      "grad_norm": 6.15336275100708,
      "learning_rate": 0.00019999840080852627,
      "logits/chosen": -0.6088159680366516,
      "logits/rejected": -0.6091991662979126,
      "logps/chosen": -10.085458755493164,
      "logps/rejected": -16.714794158935547,
      "loss": 1.2082210779190063,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6319302320480347,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.015833575278520584,
      "rewards/margins": 0.46795743703842163,
      "rewards/rejected": -0.4837909936904907,
      "step": 61,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.1059319771477388,
      "grad_norm": 4.543377876281738,
      "learning_rate": 0.00019999640183117196,
      "logits/chosen": -0.6160375475883484,
      "logits/rejected": -0.6183785796165466,
      "logps/chosen": -7.354756832122803,
      "logps/rejected": -17.332752227783203,
      "loss": 0.9280235767364502,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39269304275512695,
      "rewards/accuracies": 0.65625,
      "rewards/chosen": 0.00012398138642311096,
      "rewards/margins": 0.7065204381942749,
      "rewards/rejected": -0.7063965797424316,
      "step": 62,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.10764055742431523,
      "grad_norm": 3.700834035873413,
      "learning_rate": 0.00019999360328525325,
      "logits/chosen": -0.5955262780189514,
      "logits/rejected": -0.5960131287574768,
      "logps/chosen": -7.038027763366699,
      "logps/rejected": -19.62297248840332,
      "loss": 0.8426406383514404,
      "memory(GiB)": 46.38,
      "nll_loss": 0.304790735244751,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04702156037092209,
      "rewards/margins": 0.6756925582885742,
      "rewards/rejected": -0.6286709904670715,
      "step": 63,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.10934913770089166,
      "grad_norm": 3.616783857345581,
      "learning_rate": 0.00019999000519314722,
      "logits/chosen": -0.5933761596679688,
      "logits/rejected": -0.5956672430038452,
      "logps/chosen": -10.029403686523438,
      "logps/rejected": -19.743268966674805,
      "loss": 0.8910276889801025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4178580045700073,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.059932250529527664,
      "rewards/margins": 0.7701122164726257,
      "rewards/rejected": -0.7101800441741943,
      "step": 64,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.11105771797746809,
      "grad_norm": 2.8669040203094482,
      "learning_rate": 0.00019998560758362416,
      "logits/chosen": -0.6144078373908997,
      "logits/rejected": -0.6167560815811157,
      "logps/chosen": -10.685945510864258,
      "logps/rejected": -23.724315643310547,
      "loss": 0.7696695923805237,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3780427873134613,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1446903944015503,
      "rewards/margins": 1.0186232328414917,
      "rewards/rejected": -0.873932957649231,
      "step": 65,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.11276629825404454,
      "grad_norm": 4.312869548797607,
      "learning_rate": 0.00019998041049184719,
      "logits/chosen": -0.620762050151825,
      "logits/rejected": -0.6244264245033264,
      "logps/chosen": -11.728214263916016,
      "logps/rejected": -26.204395294189453,
      "loss": 0.7992648482322693,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4216463565826416,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.14800474047660828,
      "rewards/margins": 1.0791853666305542,
      "rewards/rejected": -0.9311806559562683,
      "step": 66,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.11447487853062097,
      "grad_norm": 7.865874290466309,
      "learning_rate": 0.00019997441395937213,
      "logits/chosen": -0.6298569440841675,
      "logits/rejected": -0.6332789659500122,
      "logps/chosen": -7.128231525421143,
      "logps/rejected": -22.970563888549805,
      "loss": 0.7514320015907288,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33478716015815735,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.05204010754823685,
      "rewards/margins": 0.9836788773536682,
      "rewards/rejected": -0.931638777256012,
      "step": 67,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.1161834588071974,
      "grad_norm": 6.099157810211182,
      "learning_rate": 0.0001999676180341471,
      "logits/chosen": -0.6536255478858948,
      "logits/rejected": -0.6564731597900391,
      "logps/chosen": -10.524028778076172,
      "logps/rejected": -19.91427993774414,
      "loss": 1.1307811737060547,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7356545925140381,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.028077581897377968,
      "rewards/margins": 0.9777989387512207,
      "rewards/rejected": -0.9497213363647461,
      "step": 68,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.11789203908377383,
      "grad_norm": 5.27453088760376,
      "learning_rate": 0.00019996002277051218,
      "logits/chosen": -0.6637449264526367,
      "logits/rejected": -0.6651371717453003,
      "logps/chosen": -9.458850860595703,
      "logps/rejected": -16.127042770385742,
      "loss": 1.4500970840454102,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8368474841117859,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.15293718874454498,
      "rewards/margins": 0.500515878200531,
      "rewards/rejected": -0.6534531116485596,
      "step": 69,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.11960061936035025,
      "grad_norm": 4.469899654388428,
      "learning_rate": 0.00019995162822919883,
      "logits/chosen": -0.6642380356788635,
      "logits/rejected": -0.664824366569519,
      "logps/chosen": -8.725358963012695,
      "logps/rejected": -16.164390563964844,
      "loss": 0.8428195714950562,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3803461194038391,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.08946336805820465,
      "rewards/margins": 0.7338303327560425,
      "rewards/rejected": -0.644366979598999,
      "step": 70,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.12130919963692668,
      "grad_norm": 4.995606422424316,
      "learning_rate": 0.00019994243447732957,
      "logits/chosen": -0.6431492567062378,
      "logits/rejected": -0.6473235487937927,
      "logps/chosen": -8.270989418029785,
      "logps/rejected": -24.483354568481445,
      "loss": 0.8698587417602539,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4563717544078827,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.07262411713600159,
      "rewards/margins": 1.0772123336791992,
      "rewards/rejected": -1.0045881271362305,
      "step": 71,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.12301777991350313,
      "grad_norm": 3.5250916481018066,
      "learning_rate": 0.00019993244158841747,
      "logits/chosen": -0.6772405505180359,
      "logits/rejected": -0.6789288520812988,
      "logps/chosen": -9.19353199005127,
      "logps/rejected": -27.746652603149414,
      "loss": 0.8238008618354797,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39661213755607605,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.09302112460136414,
      "rewards/margins": 1.367937684059143,
      "rewards/rejected": -1.274916410446167,
      "step": 72,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.12472636019007956,
      "grad_norm": 3.276311159133911,
      "learning_rate": 0.00019992164964236533,
      "logits/chosen": -0.6857064962387085,
      "logits/rejected": -0.685559868812561,
      "logps/chosen": -11.672329902648926,
      "logps/rejected": -21.510480880737305,
      "loss": 0.9018468856811523,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41466090083122253,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07735104858875275,
      "rewards/margins": 0.9590628147125244,
      "rewards/rejected": -0.8817118406295776,
      "step": 73,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.126434940466656,
      "grad_norm": 3.369281768798828,
      "learning_rate": 0.00019991005872546527,
      "logits/chosen": -0.6608355641365051,
      "logits/rejected": -0.6626392602920532,
      "logps/chosen": -7.696371078491211,
      "logps/rejected": -16.016326904296875,
      "loss": 0.8347861766815186,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3552168607711792,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13171803951263428,
      "rewards/margins": 0.7066640853881836,
      "rewards/rejected": -0.5749460458755493,
      "step": 74,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.12814352074323243,
      "grad_norm": 2.894788980484009,
      "learning_rate": 0.00019989766893039804,
      "logits/chosen": -0.6222079992294312,
      "logits/rejected": -0.6232355833053589,
      "logps/chosen": -5.741198539733887,
      "logps/rejected": -21.31194305419922,
      "loss": 0.7077224850654602,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2760768234729767,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.08473706245422363,
      "rewards/margins": 0.9282934665679932,
      "rewards/rejected": -0.8435564041137695,
      "step": 75,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.12985210101980885,
      "grad_norm": 2.1586077213287354,
      "learning_rate": 0.00019988448035623203,
      "logits/chosen": -0.6409361958503723,
      "logits/rejected": -0.6418861150741577,
      "logps/chosen": -7.263876438140869,
      "logps/rejected": -17.95524024963379,
      "loss": 0.8056262731552124,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2723458409309387,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.10251796990633011,
      "rewards/margins": 0.5812361240386963,
      "rewards/rejected": -0.4787181317806244,
      "step": 76,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1315606812963853,
      "grad_norm": 3.0591204166412354,
      "learning_rate": 0.00019987049310842272,
      "logits/chosen": -0.6324824690818787,
      "logits/rejected": -0.6336201429367065,
      "logps/chosen": -10.325702667236328,
      "logps/rejected": -15.849100112915039,
      "loss": 0.8897943496704102,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3855420649051666,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16009055078029633,
      "rewards/margins": 0.6010256409645081,
      "rewards/rejected": -0.4409351050853729,
      "step": 77,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1332692615729617,
      "grad_norm": 2.668419122695923,
      "learning_rate": 0.00019985570729881184,
      "logits/chosen": -0.6462000012397766,
      "logits/rejected": -0.6490585207939148,
      "logps/chosen": -5.475035667419434,
      "logps/rejected": -20.422216415405273,
      "loss": 0.7187508344650269,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26306530833244324,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.13646253943443298,
      "rewards/margins": 0.6754783391952515,
      "rewards/rejected": -0.5390157699584961,
      "step": 78,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.13497784184953815,
      "grad_norm": 4.360518932342529,
      "learning_rate": 0.00019984012304562622,
      "logits/chosen": -0.6780133247375488,
      "logits/rejected": -0.6758167147636414,
      "logps/chosen": -11.310664176940918,
      "logps/rejected": -15.12096881866455,
      "loss": 0.9152082800865173,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45073550939559937,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15347006916999817,
      "rewards/margins": 0.7282993197441101,
      "rewards/rejected": -0.5748292207717896,
      "step": 79,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.1366864221261146,
      "grad_norm": 2.7597768306732178,
      "learning_rate": 0.00019982374047347723,
      "logits/chosen": -0.6909894943237305,
      "logits/rejected": -0.694047212600708,
      "logps/chosen": -6.004232406616211,
      "logps/rejected": -23.221824645996094,
      "loss": 0.6792643666267395,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31354260444641113,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21520498394966125,
      "rewards/margins": 1.1965129375457764,
      "rewards/rejected": -0.9813080430030823,
      "step": 80,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.138395002402691,
      "grad_norm": 4.00593900680542,
      "learning_rate": 0.00019980655971335945,
      "logits/chosen": -0.666761577129364,
      "logits/rejected": -0.6698122620582581,
      "logps/chosen": -11.200495719909668,
      "logps/rejected": -20.89803123474121,
      "loss": 1.0373343229293823,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5717941522598267,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.03891691192984581,
      "rewards/margins": 0.8691489696502686,
      "rewards/rejected": -0.9080657958984375,
      "step": 81,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.14010358267926745,
      "grad_norm": 3.7606523036956787,
      "learning_rate": 0.00019978858090264975,
      "logits/chosen": -0.6808147430419922,
      "logits/rejected": -0.683088481426239,
      "logps/chosen": -8.414874076843262,
      "logps/rejected": -28.505592346191406,
      "loss": 0.7084274888038635,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3828541338443756,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1215677559375763,
      "rewards/margins": 1.7334877252578735,
      "rewards/rejected": -1.6119199991226196,
      "step": 82,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.14181216295584387,
      "grad_norm": 4.108179569244385,
      "learning_rate": 0.0001997698041851063,
      "logits/chosen": -0.6897909045219421,
      "logits/rejected": -0.6913143396377563,
      "logps/chosen": -4.813051223754883,
      "logps/rejected": -26.06230926513672,
      "loss": 0.572483241558075,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24444976449012756,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15832601487636566,
      "rewards/margins": 1.6045297384262085,
      "rewards/rejected": -1.4462038278579712,
      "step": 83,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1435207432324203,
      "grad_norm": 5.49106502532959,
      "learning_rate": 0.00019975022971086714,
      "logits/chosen": -0.6466387510299683,
      "logits/rejected": -0.6520243883132935,
      "logps/chosen": -7.595500946044922,
      "logps/rejected": -33.218482971191406,
      "loss": 0.9090702533721924,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5250993371009827,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.13341139256954193,
      "rewards/margins": 1.5776695013046265,
      "rewards/rejected": -1.444258213043213,
      "step": 84,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.14522932350899675,
      "grad_norm": 4.539761543273926,
      "learning_rate": 0.00019972985763644932,
      "logits/chosen": -0.6421160101890564,
      "logits/rejected": -0.6411360502243042,
      "logps/chosen": -7.4049482345581055,
      "logps/rejected": -21.37047576904297,
      "loss": 0.8407540917396545,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4470757246017456,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1697394847869873,
      "rewards/margins": 1.345821738243103,
      "rewards/rejected": -1.1760823726654053,
      "step": 85,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.14693790378557317,
      "grad_norm": 4.877723217010498,
      "learning_rate": 0.00019970868812474736,
      "logits/chosen": -0.6163697242736816,
      "logits/rejected": -0.6189360022544861,
      "logps/chosen": -12.07491683959961,
      "logps/rejected": -29.372955322265625,
      "loss": 0.848598301410675,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49112260341644287,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1017676591873169,
      "rewards/margins": 1.7578887939453125,
      "rewards/rejected": -1.6561211347579956,
      "step": 86,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.14864648406214961,
      "grad_norm": 5.431716442108154,
      "learning_rate": 0.00019968672134503213,
      "logits/chosen": -0.6225081086158752,
      "logits/rejected": -0.6252766251564026,
      "logps/chosen": -7.893375873565674,
      "logps/rejected": -24.980260848999023,
      "loss": 0.7993619441986084,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3846016228199005,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.16642117500305176,
      "rewards/margins": 1.3823862075805664,
      "rewards/rejected": -1.215964913368225,
      "step": 87,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.15035506433872603,
      "grad_norm": 3.7740750312805176,
      "learning_rate": 0.00019966395747294938,
      "logits/chosen": -0.5984699130058289,
      "logits/rejected": -0.6030604839324951,
      "logps/chosen": -5.385066032409668,
      "logps/rejected": -25.168312072753906,
      "loss": 0.6654453873634338,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3348524868488312,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16900494694709778,
      "rewards/margins": 1.3018345832824707,
      "rewards/rejected": -1.1328295469284058,
      "step": 88,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.15206364461530247,
      "grad_norm": 3.6669890880584717,
      "learning_rate": 0.0001996403966905185,
      "logits/chosen": -0.6115779280662537,
      "logits/rejected": -0.615218997001648,
      "logps/chosen": -8.385238647460938,
      "logps/rejected": -20.905569076538086,
      "loss": 0.8084090948104858,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39410629868507385,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.07257324457168579,
      "rewards/margins": 1.0608566999435425,
      "rewards/rejected": -0.9882834553718567,
      "step": 89,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.15377222489187892,
      "grad_norm": 8.954024314880371,
      "learning_rate": 0.00019961603918613077,
      "logits/chosen": -0.6033557653427124,
      "logits/rejected": -0.6063033938407898,
      "logps/chosen": -7.85408878326416,
      "logps/rejected": -24.28641700744629,
      "loss": 0.8988556861877441,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4972013235092163,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1499406397342682,
      "rewards/margins": 1.2390244007110596,
      "rewards/rejected": -1.0890836715698242,
      "step": 90,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.15548080516845533,
      "grad_norm": 2.68121075630188,
      "learning_rate": 0.00019959088515454827,
      "logits/chosen": -0.6473177075386047,
      "logits/rejected": -0.6510601043701172,
      "logps/chosen": -6.986569404602051,
      "logps/rejected": -28.0955810546875,
      "loss": 0.6454746723175049,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3593181073665619,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.310781866312027,
      "rewards/margins": 1.7279632091522217,
      "rewards/rejected": -1.417181372642517,
      "step": 91,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.15718938544503178,
      "grad_norm": 7.000541687011719,
      "learning_rate": 0.0001995649347969019,
      "logits/chosen": -0.7174542546272278,
      "logits/rejected": -0.7201290726661682,
      "logps/chosen": -11.954474449157715,
      "logps/rejected": -24.34893226623535,
      "loss": 1.0948961973190308,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6069883704185486,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.005067221820354462,
      "rewards/margins": 1.2273277044296265,
      "rewards/rejected": -1.232394814491272,
      "step": 92,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1588979657216082,
      "grad_norm": 4.3811211585998535,
      "learning_rate": 0.00019953818832069,
      "logits/chosen": -0.6739280223846436,
      "logits/rejected": -0.6795160174369812,
      "logps/chosen": -6.881966590881348,
      "logps/rejected": -37.4478645324707,
      "loss": 0.6469016075134277,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40633073449134827,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.20596878230571747,
      "rewards/margins": 2.5138516426086426,
      "rewards/rejected": -2.3078830242156982,
      "step": 93,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.16060654599818464,
      "grad_norm": 12.26343822479248,
      "learning_rate": 0.00019951064593977668,
      "logits/chosen": -0.6859129667282104,
      "logits/rejected": -0.6899083852767944,
      "logps/chosen": -13.38476848602295,
      "logps/rejected": -30.595182418823242,
      "loss": 1.2918721437454224,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6223256587982178,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.30435577034950256,
      "rewards/margins": 1.605468988418579,
      "rewards/rejected": -1.9098248481750488,
      "step": 94,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.16231512627476108,
      "grad_norm": 6.570938587188721,
      "learning_rate": 0.00019948230787439017,
      "logits/chosen": -0.7028453946113586,
      "logits/rejected": -0.7050482034683228,
      "logps/chosen": -6.761815071105957,
      "logps/rejected": -27.314367294311523,
      "loss": 0.9538392424583435,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39429235458374023,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.012213893234729767,
      "rewards/margins": 1.6957728862762451,
      "rewards/rejected": -1.6835590600967407,
      "step": 95,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.1640237065513375,
      "grad_norm": 2.8915793895721436,
      "learning_rate": 0.0001994531743511208,
      "logits/chosen": -0.6953604817390442,
      "logits/rejected": -0.6982015371322632,
      "logps/chosen": -7.192444801330566,
      "logps/rejected": -26.632644653320312,
      "loss": 0.6771315336227417,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3128400444984436,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.21012118458747864,
      "rewards/margins": 1.6979337930679321,
      "rewards/rejected": -1.4878127574920654,
      "step": 96,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.16573228682791394,
      "grad_norm": 2.7815051078796387,
      "learning_rate": 0.00019942324560291952,
      "logits/chosen": -0.6696122884750366,
      "logits/rejected": -0.6713677644729614,
      "logps/chosen": -4.250362396240234,
      "logps/rejected": -22.2177791595459,
      "loss": 0.6821787357330322,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26163250207901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.09910638630390167,
      "rewards/margins": 1.2715519666671753,
      "rewards/rejected": -1.1724457740783691,
      "step": 97,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.16744086710449035,
      "grad_norm": 2.4311976432800293,
      "learning_rate": 0.00019939252186909574,
      "logits/chosen": -0.6763203740119934,
      "logits/rejected": -0.6752580404281616,
      "logps/chosen": -6.972185134887695,
      "logps/rejected": -21.0667667388916,
      "loss": 0.5966892838478088,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2420358657836914,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14732374250888824,
      "rewards/margins": 1.0790168046951294,
      "rewards/rejected": -0.9316931366920471,
      "step": 98,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.1691494473810668,
      "grad_norm": 5.822193622589111,
      "learning_rate": 0.00019936100339531564,
      "logits/chosen": -0.6894733905792236,
      "logits/rejected": -0.6909453272819519,
      "logps/chosen": -6.009974479675293,
      "logps/rejected": -21.0426025390625,
      "loss": 0.7569796442985535,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4093407690525055,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.26579749584198,
      "rewards/margins": 1.3290454149246216,
      "rewards/rejected": -1.0632479190826416,
      "step": 99,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.1708580276576432,
      "grad_norm": 3.2212934494018555,
      "learning_rate": 0.0001993286904336001,
      "logits/chosen": -0.7202757000923157,
      "logits/rejected": -0.7197086215019226,
      "logps/chosen": -6.131860733032227,
      "logps/rejected": -21.27735137939453,
      "loss": 0.622944712638855,
      "memory(GiB)": 46.38,
      "nll_loss": 0.28273510932922363,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26464149355888367,
      "rewards/margins": 1.1921066045761108,
      "rewards/rejected": -0.9274651408195496,
      "step": 100,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.17256660793421966,
      "grad_norm": 5.42195463180542,
      "learning_rate": 0.00019929558324232268,
      "logits/chosen": -0.7189818620681763,
      "logits/rejected": -0.720359742641449,
      "logps/chosen": -8.62668228149414,
      "logps/rejected": -19.183561325073242,
      "loss": 0.9257504940032959,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5092519521713257,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09434813261032104,
      "rewards/margins": 0.8864970803260803,
      "rewards/rejected": -0.7921488881111145,
      "step": 101,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.1742751882107961,
      "grad_norm": 6.533127307891846,
      "learning_rate": 0.0001992616820862076,
      "logits/chosen": -0.6889673471450806,
      "logits/rejected": -0.6902090907096863,
      "logps/chosen": -9.093452453613281,
      "logps/rejected": -19.451143264770508,
      "loss": 0.9490825533866882,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45314493775367737,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03901132568717003,
      "rewards/margins": 0.7304368019104004,
      "rewards/rejected": -0.6914253830909729,
      "step": 102,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.17598376848737252,
      "grad_norm": 3.991544485092163,
      "learning_rate": 0.00019922698723632767,
      "logits/chosen": -0.7347920536994934,
      "logits/rejected": -0.7329245209693909,
      "logps/chosen": -10.086681365966797,
      "logps/rejected": -24.030868530273438,
      "loss": 0.8895858526229858,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5397864580154419,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.12226971983909607,
      "rewards/margins": 1.3651487827301025,
      "rewards/rejected": -1.2428791522979736,
      "step": 103,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.17769234876394896,
      "grad_norm": 4.0011162757873535,
      "learning_rate": 0.0001991914989701019,
      "logits/chosen": -0.7434298992156982,
      "logits/rejected": -0.7457666993141174,
      "logps/chosen": -8.058287620544434,
      "logps/rejected": -20.040809631347656,
      "loss": 0.7681674957275391,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3557548522949219,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.12300814688205719,
      "rewards/margins": 1.063488483428955,
      "rewards/rejected": -0.9404802918434143,
      "step": 104,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.17940092904052538,
      "grad_norm": 3.329845905303955,
      "learning_rate": 0.00019915521757129354,
      "logits/chosen": -0.7461959719657898,
      "logits/rejected": -0.7488771677017212,
      "logps/chosen": -7.418200969696045,
      "logps/rejected": -25.79469108581543,
      "loss": 0.6970200538635254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36587315797805786,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12871423363685608,
      "rewards/margins": 1.5802878141403198,
      "rewards/rejected": -1.4515737295150757,
      "step": 105,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.18110950931710182,
      "grad_norm": 4.27202033996582,
      "learning_rate": 0.00019911814333000772,
      "logits/chosen": -0.7183383703231812,
      "logits/rejected": -0.7201208472251892,
      "logps/chosen": -9.424932479858398,
      "logps/rejected": -25.1419677734375,
      "loss": 0.8681972622871399,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4449956715106964,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.04691670835018158,
      "rewards/margins": 1.2326714992523193,
      "rewards/rejected": -1.1857548952102661,
      "step": 106,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.18281808959367826,
      "grad_norm": 4.578512191772461,
      "learning_rate": 0.00019908027654268903,
      "logits/chosen": -0.7675899267196655,
      "logits/rejected": -0.7682605385780334,
      "logps/chosen": -8.889389038085938,
      "logps/rejected": -24.62353515625,
      "loss": 0.8598517179489136,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40520450472831726,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13606908917427063,
      "rewards/margins": 1.3364840745925903,
      "rewards/rejected": -1.200415015220642,
      "step": 107,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.18452666987025468,
      "grad_norm": 3.067852020263672,
      "learning_rate": 0.00019904161751211938,
      "logits/chosen": -0.7350285649299622,
      "logits/rejected": -0.7360118627548218,
      "logps/chosen": -12.491768836975098,
      "logps/rejected": -23.78077507019043,
      "loss": 0.8337110280990601,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4792723059654236,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0716579332947731,
      "rewards/margins": 1.2602936029434204,
      "rewards/rejected": -1.1886355876922607,
      "step": 108,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.18623525014683112,
      "grad_norm": 3.2055752277374268,
      "learning_rate": 0.00019900216654741528,
      "logits/chosen": -0.7312414646148682,
      "logits/rejected": -0.7316892147064209,
      "logps/chosen": -8.460771560668945,
      "logps/rejected": -21.30421257019043,
      "loss": 0.7292563319206238,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3923528492450714,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2296271026134491,
      "rewards/margins": 1.3531551361083984,
      "rewards/rejected": -1.1235278844833374,
      "step": 109,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.18794383042340754,
      "grad_norm": 4.93084716796875,
      "learning_rate": 0.0001989619239640256,
      "logits/chosen": -0.7337685823440552,
      "logits/rejected": -0.7344739437103271,
      "logps/chosen": -11.766962051391602,
      "logps/rejected": -22.844139099121094,
      "loss": 0.8851971626281738,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4795154333114624,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06328105181455612,
      "rewards/margins": 1.133680820465088,
      "rewards/rejected": -1.0703996419906616,
      "step": 110,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.18965241069998398,
      "grad_norm": 10.518948554992676,
      "learning_rate": 0.00019892089008372897,
      "logits/chosen": -0.709618330001831,
      "logits/rejected": -0.7111095190048218,
      "logps/chosen": -8.326457977294922,
      "logps/rejected": -27.638351440429688,
      "loss": 0.9548771977424622,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5345268249511719,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.045853935182094574,
      "rewards/margins": 1.551784634590149,
      "rewards/rejected": -1.5059306621551514,
      "step": 111,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.19136099097656042,
      "grad_norm": 7.465288162231445,
      "learning_rate": 0.0001988790652346312,
      "logits/chosen": -0.7286843657493591,
      "logits/rejected": -0.729621946811676,
      "logps/chosen": -8.483978271484375,
      "logps/rejected": -28.15561294555664,
      "loss": 0.922063410282135,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5344014167785645,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.13854525983333588,
      "rewards/margins": 1.4421772956848145,
      "rewards/rejected": -1.3036320209503174,
      "step": 112,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.19306957125313684,
      "grad_norm": 8.20088005065918,
      "learning_rate": 0.0001988364497511627,
      "logits/chosen": -0.740546703338623,
      "logits/rejected": -0.7427754402160645,
      "logps/chosen": -8.096268653869629,
      "logps/rejected": -22.899229049682617,
      "loss": 0.9872632622718811,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5750774145126343,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18856163322925568,
      "rewards/margins": 1.2329988479614258,
      "rewards/rejected": -1.0444371700286865,
      "step": 113,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.19477815152971328,
      "grad_norm": 3.4554946422576904,
      "learning_rate": 0.0001987930439740757,
      "logits/chosen": -0.699242115020752,
      "logits/rejected": -0.7016286849975586,
      "logps/chosen": -11.758835792541504,
      "logps/rejected": -22.287220001220703,
      "loss": 0.7449758052825928,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4075641632080078,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1979522556066513,
      "rewards/margins": 1.2950595617294312,
      "rewards/rejected": -1.097107172012329,
      "step": 114,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.1964867318062897,
      "grad_norm": 2.8801109790802,
      "learning_rate": 0.00019874884825044165,
      "logits/chosen": -0.673065185546875,
      "logits/rejected": -0.6760713458061218,
      "logps/chosen": -6.409548759460449,
      "logps/rejected": -16.89990234375,
      "loss": 0.7451252937316895,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31410443782806396,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.19196031987667084,
      "rewards/margins": 0.9307445287704468,
      "rewards/rejected": -0.7387841939926147,
      "step": 115,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.19819531208286614,
      "grad_norm": 2.597715377807617,
      "learning_rate": 0.00019870386293364836,
      "logits/chosen": -0.642209529876709,
      "logits/rejected": -0.6442127227783203,
      "logps/chosen": -6.406985282897949,
      "logps/rejected": -19.21172523498535,
      "loss": 0.7881473898887634,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3131154477596283,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.14077159762382507,
      "rewards/margins": 0.9926955699920654,
      "rewards/rejected": -0.851923942565918,
      "step": 116,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.1999038923594426,
      "grad_norm": 2.9220879077911377,
      "learning_rate": 0.00019865808838339727,
      "logits/chosen": -0.6920750141143799,
      "logits/rejected": -0.6941624283790588,
      "logps/chosen": -7.081650733947754,
      "logps/rejected": -24.63327407836914,
      "loss": 0.6955921649932861,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29941731691360474,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.014344778843224049,
      "rewards/margins": 1.21220862865448,
      "rewards/rejected": -1.1978638172149658,
      "step": 117,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.201612472636019,
      "grad_norm": 2.535001277923584,
      "learning_rate": 0.00019861152496570043,
      "logits/chosen": -0.6474766135215759,
      "logits/rejected": -0.6474269032478333,
      "logps/chosen": -4.845932483673096,
      "logps/rejected": -30.43771743774414,
      "loss": 0.549997866153717,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2411210685968399,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22833456099033356,
      "rewards/margins": 1.7972755432128906,
      "rewards/rejected": -1.5689408779144287,
      "step": 118,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.20332105291259545,
      "grad_norm": 2.592416286468506,
      "learning_rate": 0.00019856417305287772,
      "logits/chosen": -0.6813372373580933,
      "logits/rejected": -0.684921383857727,
      "logps/chosen": -8.64324951171875,
      "logps/rejected": -25.918020248413086,
      "loss": 0.6950156092643738,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34918904304504395,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.27043017745018005,
      "rewards/margins": 1.5927720069885254,
      "rewards/rejected": -1.322341799736023,
      "step": 119,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.20502963318917186,
      "grad_norm": 3.0879509449005127,
      "learning_rate": 0.00019851603302355373,
      "logits/chosen": -0.7103255987167358,
      "logits/rejected": -0.7141609191894531,
      "logps/chosen": -7.374659061431885,
      "logps/rejected": -27.09424591064453,
      "loss": 0.7110822200775146,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3552224040031433,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15158146619796753,
      "rewards/margins": 1.6895453929901123,
      "rewards/rejected": -1.5379639863967896,
      "step": 120,
      "train_speed(iter/s)": 0.005499
    },
    {
      "epoch": 0.2067382134657483,
      "grad_norm": 3.703174591064453,
      "learning_rate": 0.00019846710526265487,
      "logits/chosen": -0.7088285088539124,
      "logits/rejected": -0.7127594947814941,
      "logps/chosen": -6.05606746673584,
      "logps/rejected": -27.269733428955078,
      "loss": 0.619713306427002,
      "memory(GiB)": 46.38,
      "nll_loss": 0.333889901638031,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2516608238220215,
      "rewards/margins": 1.8369193077087402,
      "rewards/rejected": -1.5852584838867188,
      "step": 121,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.20844679374232475,
      "grad_norm": 4.171956539154053,
      "learning_rate": 0.00019841739016140624,
      "logits/chosen": -0.7836604118347168,
      "logits/rejected": -0.7841136455535889,
      "logps/chosen": -7.950913429260254,
      "logps/rejected": -26.353294372558594,
      "loss": 0.7199419736862183,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3016027510166168,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.03286279737949371,
      "rewards/margins": 1.4103941917419434,
      "rewards/rejected": -1.3775315284729004,
      "step": 122,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.21015537401890116,
      "grad_norm": 5.59217643737793,
      "learning_rate": 0.0001983668881173284,
      "logits/chosen": -0.7123353481292725,
      "logits/rejected": -0.7176805734634399,
      "logps/chosen": -6.9763007164001465,
      "logps/rejected": -40.705894470214844,
      "loss": 0.5055665969848633,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2759905159473419,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13582277297973633,
      "rewards/margins": 2.353527069091797,
      "rewards/rejected": -2.2177045345306396,
      "step": 123,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.2118639542954776,
      "grad_norm": 7.052844524383545,
      "learning_rate": 0.00019831559953423442,
      "logits/chosen": -0.7205808758735657,
      "logits/rejected": -0.7224647402763367,
      "logps/chosen": -8.208627700805664,
      "logps/rejected": -23.56304359436035,
      "loss": 0.8278493285179138,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3575640022754669,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0022782105952501297,
      "rewards/margins": 1.129638910293579,
      "rewards/rejected": -1.1273607015609741,
      "step": 124,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.21357253457205402,
      "grad_norm": 2.853487491607666,
      "learning_rate": 0.00019826352482222638,
      "logits/chosen": -0.7876097559928894,
      "logits/rejected": -0.7900496125221252,
      "logps/chosen": -6.370059013366699,
      "logps/rejected": -29.205093383789062,
      "loss": 0.553210437297821,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24043366312980652,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.03330150991678238,
      "rewards/margins": 1.9821114540100098,
      "rewards/rejected": -1.9488096237182617,
      "step": 125,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.21528111484863047,
      "grad_norm": 3.3915181159973145,
      "learning_rate": 0.00019821066439769234,
      "logits/chosen": -0.7244046926498413,
      "logits/rejected": -0.7288062572479248,
      "logps/chosen": -7.876413822174072,
      "logps/rejected": -35.36836242675781,
      "loss": 0.5778641104698181,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34048542380332947,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.027921676635742188,
      "rewards/margins": 2.1973161697387695,
      "rewards/rejected": -2.225238084793091,
      "step": 126,
      "train_speed(iter/s)": 0.0055
    },
    {
      "epoch": 0.2169896951252069,
      "grad_norm": 2.667902946472168,
      "learning_rate": 0.00019815701868330284,
      "logits/chosen": -0.7135522365570068,
      "logits/rejected": -0.7176554799079895,
      "logps/chosen": -11.242607116699219,
      "logps/rejected": -33.319515228271484,
      "loss": 0.7801873087882996,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49927234649658203,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23649244010448456,
      "rewards/margins": 2.445638656616211,
      "rewards/rejected": -2.209146022796631,
      "step": 127,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.21869827540178333,
      "grad_norm": 9.078632354736328,
      "learning_rate": 0.00019810258810800752,
      "logits/chosen": -0.7386202812194824,
      "logits/rejected": -0.7396747469902039,
      "logps/chosen": -9.633708000183105,
      "logps/rejected": -30.443008422851562,
      "loss": 0.7859803438186646,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49833089113235474,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.04507865756750107,
      "rewards/margins": 1.9784715175628662,
      "rewards/rejected": -1.933392882347107,
      "step": 128,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.22040685567835977,
      "grad_norm": 13.282059669494629,
      "learning_rate": 0.0001980473731070319,
      "logits/chosen": -0.7503476142883301,
      "logits/rejected": -0.755096971988678,
      "logps/chosen": -7.456392765045166,
      "logps/rejected": -29.754714965820312,
      "loss": 0.8027007579803467,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4415196180343628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14411740005016327,
      "rewards/margins": 2.093838691711426,
      "rewards/rejected": -1.949721336364746,
      "step": 129,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.22211543595493619,
      "grad_norm": 5.136019706726074,
      "learning_rate": 0.0001979913741218736,
      "logits/chosen": -0.7441588640213013,
      "logits/rejected": -0.7470377683639526,
      "logps/chosen": -11.379145622253418,
      "logps/rejected": -36.20026397705078,
      "loss": 0.9255682229995728,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5846636891365051,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.014721579849720001,
      "rewards/margins": 2.245670795440674,
      "rewards/rejected": -2.2309491634368896,
      "step": 130,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.22382401623151263,
      "grad_norm": 5.494217395782471,
      "learning_rate": 0.00019793459160029903,
      "logits/chosen": -0.7520757913589478,
      "logits/rejected": -0.7574396133422852,
      "logps/chosen": -6.406137466430664,
      "logps/rejected": -42.744041442871094,
      "loss": 0.6142405867576599,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3836834132671356,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.19515469670295715,
      "rewards/margins": 3.062283992767334,
      "rewards/rejected": -2.867129325866699,
      "step": 131,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.22553259650808907,
      "grad_norm": 4.463568687438965,
      "learning_rate": 0.00019787702599633971,
      "logits/chosen": -0.7327495217323303,
      "logits/rejected": -0.738735556602478,
      "logps/chosen": -6.503672122955322,
      "logps/rejected": -39.14170837402344,
      "loss": 0.7878039479255676,
      "memory(GiB)": 46.38,
      "nll_loss": 0.37348490953445435,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.034972380846738815,
      "rewards/margins": 2.3170716762542725,
      "rewards/rejected": -2.352044105529785,
      "step": 132,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.2272411767846655,
      "grad_norm": 7.909404754638672,
      "learning_rate": 0.0001978186777702887,
      "logits/chosen": -0.6934617757797241,
      "logits/rejected": -0.6948121786117554,
      "logps/chosen": -7.257785797119141,
      "logps/rejected": -27.908708572387695,
      "loss": 0.7475972175598145,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3585331439971924,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.036370277404785156,
      "rewards/margins": 1.666322946548462,
      "rewards/rejected": -1.6299527883529663,
      "step": 133,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.22894975706124193,
      "grad_norm": 14.575544357299805,
      "learning_rate": 0.00019775954738869683,
      "logits/chosen": -0.6997110843658447,
      "logits/rejected": -0.7026513814926147,
      "logps/chosen": -7.946556091308594,
      "logps/rejected": -20.762996673583984,
      "loss": 1.056870460510254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6803016066551208,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.16680209338665009,
      "rewards/margins": 1.3951897621154785,
      "rewards/rejected": -1.2283875942230225,
      "step": 134,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.23065833733781835,
      "grad_norm": 6.814887046813965,
      "learning_rate": 0.00019769963532436911,
      "logits/chosen": -0.661209762096405,
      "logits/rejected": -0.6659980416297913,
      "logps/chosen": -8.108284950256348,
      "logps/rejected": -26.44216537475586,
      "loss": 1.0938985347747803,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7105523347854614,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.010204421356320381,
      "rewards/margins": 1.3176696300506592,
      "rewards/rejected": -1.3074650764465332,
      "step": 135,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.2323669176143948,
      "grad_norm": 5.490352153778076,
      "learning_rate": 0.00019763894205636072,
      "logits/chosen": -0.671430230140686,
      "logits/rejected": -0.673378586769104,
      "logps/chosen": -9.945783615112305,
      "logps/rejected": -22.1285343170166,
      "loss": 0.976132333278656,
      "memory(GiB)": 46.38,
      "nll_loss": 0.543598473072052,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1351984441280365,
      "rewards/margins": 1.1337149143218994,
      "rewards/rejected": -0.9985164403915405,
      "step": 136,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.23407549789097123,
      "grad_norm": 5.260707378387451,
      "learning_rate": 0.00019757746806997348,
      "logits/chosen": -0.6116827130317688,
      "logits/rejected": -0.6123840808868408,
      "logps/chosen": -8.495226860046387,
      "logps/rejected": -17.80582046508789,
      "loss": 0.9422957897186279,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3756696879863739,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.009481308050453663,
      "rewards/margins": 0.6523244380950928,
      "rewards/rejected": -0.6618058085441589,
      "step": 137,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.23578407816754765,
      "grad_norm": 2.1559102535247803,
      "learning_rate": 0.00019751521385675172,
      "logits/chosen": -0.6613444089889526,
      "logits/rejected": -0.6647210121154785,
      "logps/chosen": -4.250008583068848,
      "logps/rejected": -25.66912078857422,
      "loss": 0.5518596172332764,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1721974015235901,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.17000286281108856,
      "rewards/margins": 1.491791009902954,
      "rewards/rejected": -1.3217883110046387,
      "step": 138,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.2374926584441241,
      "grad_norm": 2.9551846981048584,
      "learning_rate": 0.00019745217991447848,
      "logits/chosen": -0.6719025373458862,
      "logits/rejected": -0.6741494536399841,
      "logps/chosen": -8.146540641784668,
      "logps/rejected": -28.308319091796875,
      "loss": 0.7604531049728394,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34055855870246887,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": 0.1295825093984604,
      "rewards/margins": 1.5078970193862915,
      "rewards/rejected": -1.37831449508667,
      "step": 139,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.2392012387207005,
      "grad_norm": 5.525161266326904,
      "learning_rate": 0.0001973883667471715,
      "logits/chosen": -0.6694546937942505,
      "logits/rejected": -0.6737231612205505,
      "logps/chosen": -10.553884506225586,
      "logps/rejected": -32.512393951416016,
      "loss": 0.822769284248352,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3502211570739746,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.01343974657356739,
      "rewards/margins": 1.7657444477081299,
      "rewards/rejected": -1.7523046731948853,
      "step": 140,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.24090981899727695,
      "grad_norm": 2.4956538677215576,
      "learning_rate": 0.00019732377486507915,
      "logits/chosen": -0.6616899371147156,
      "logits/rejected": -0.6654747724533081,
      "logps/chosen": -6.116077423095703,
      "logps/rejected": -22.98920440673828,
      "loss": 0.7530378103256226,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33646857738494873,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09931784868240356,
      "rewards/margins": 1.2228065729141235,
      "rewards/rejected": -1.1234887838363647,
      "step": 141,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.24261839927385337,
      "grad_norm": 3.0544512271881104,
      "learning_rate": 0.00019725840478467636,
      "logits/chosen": -0.6879534125328064,
      "logits/rejected": -0.6862825155258179,
      "logps/chosen": -8.299098014831543,
      "logps/rejected": -18.718904495239258,
      "loss": 0.7917457818984985,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31696975231170654,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.11313430219888687,
      "rewards/margins": 0.9731096029281616,
      "rewards/rejected": -0.859975278377533,
      "step": 142,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.2443269795504298,
      "grad_norm": 2.7635226249694824,
      "learning_rate": 0.00019719225702866062,
      "logits/chosen": -0.7383114099502563,
      "logits/rejected": -0.736755907535553,
      "logps/chosen": -5.972444534301758,
      "logps/rejected": -27.483884811401367,
      "loss": 0.6677893400192261,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3221934735774994,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15736974775791168,
      "rewards/margins": 1.7971253395080566,
      "rewards/rejected": -1.639755368232727,
      "step": 143,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.24603555982700626,
      "grad_norm": 5.206491470336914,
      "learning_rate": 0.0001971253321259476,
      "logits/chosen": -0.7135074138641357,
      "logits/rejected": -0.7155020833015442,
      "logps/chosen": -9.462960243225098,
      "logps/rejected": -22.44765853881836,
      "loss": 0.8617446422576904,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4634811282157898,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.1327909231185913,
      "rewards/margins": 1.3640327453613281,
      "rewards/rejected": -1.2312417030334473,
      "step": 144,
      "train_speed(iter/s)": 0.005502
    },
    {
      "epoch": 0.24774414010358267,
      "grad_norm": 2.5136232376098633,
      "learning_rate": 0.00019705763061166696,
      "logits/chosen": -0.7309613227844238,
      "logits/rejected": -0.7339869141578674,
      "logps/chosen": -8.707137107849121,
      "logps/rejected": -30.160343170166016,
      "loss": 0.6045644879341125,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31675195693969727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1917949914932251,
      "rewards/margins": 2.2541422843933105,
      "rewards/rejected": -2.062347412109375,
      "step": 145,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.24945272038015912,
      "grad_norm": 3.6773223876953125,
      "learning_rate": 0.00019698915302715836,
      "logits/chosen": -0.7318214774131775,
      "logits/rejected": -0.7343602180480957,
      "logps/chosen": -6.429262161254883,
      "logps/rejected": -31.925209045410156,
      "loss": 0.6257779598236084,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31483885645866394,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1830574870109558,
      "rewards/margins": 1.988149642944336,
      "rewards/rejected": -1.8050918579101562,
      "step": 146,
      "train_speed(iter/s)": 0.005505
    },
    {
      "epoch": 0.25116130065673553,
      "grad_norm": 9.478280067443848,
      "learning_rate": 0.00019691989991996663,
      "logits/chosen": -0.7461144924163818,
      "logits/rejected": -0.7457807660102844,
      "logps/chosen": -10.189486503601074,
      "logps/rejected": -34.494956970214844,
      "loss": 0.7869279980659485,
      "memory(GiB)": 46.38,
      "nll_loss": 0.47246837615966797,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1757950335741043,
      "rewards/margins": 2.238186836242676,
      "rewards/rejected": -2.062391996383667,
      "step": 147,
      "train_speed(iter/s)": 0.005504
    },
    {
      "epoch": 0.252869880933312,
      "grad_norm": 7.141703128814697,
      "learning_rate": 0.00019684987184383776,
      "logits/chosen": -0.7700251340866089,
      "logits/rejected": -0.7725012302398682,
      "logps/chosen": -6.46027946472168,
      "logps/rejected": -34.821231842041016,
      "loss": 0.692828357219696,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3844330608844757,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.12604108452796936,
      "rewards/margins": 2.6629996299743652,
      "rewards/rejected": -2.5369582176208496,
      "step": 148,
      "train_speed(iter/s)": 0.005505
    },
    {
      "epoch": 0.2545784612098884,
      "grad_norm": 14.823970794677734,
      "learning_rate": 0.00019677906935871447,
      "logits/chosen": -0.7374511361122131,
      "logits/rejected": -0.7389614582061768,
      "logps/chosen": -10.582510948181152,
      "logps/rejected": -26.03261375427246,
      "loss": 1.33296537399292,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6253257989883423,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.18944962322711945,
      "rewards/margins": 1.5741108655929565,
      "rewards/rejected": -1.7635605335235596,
      "step": 149,
      "train_speed(iter/s)": 0.005504
    },
    {
      "epoch": 0.25628704148646486,
      "grad_norm": 13.657255172729492,
      "learning_rate": 0.00019670749303073146,
      "logits/chosen": -0.7172877192497253,
      "logits/rejected": -0.7211941480636597,
      "logps/chosen": -6.75715970993042,
      "logps/rejected": -41.68476867675781,
      "loss": 0.7531163692474365,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4120977818965912,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10011129081249237,
      "rewards/margins": 3.0644404888153076,
      "rewards/rejected": -2.964329481124878,
      "step": 150,
      "train_speed(iter/s)": 0.005503
    },
    {
      "epoch": 0.25799562176304125,
      "grad_norm": 4.298210144042969,
      "learning_rate": 0.00019663514343221117,
      "logits/chosen": -0.6899698972702026,
      "logits/rejected": -0.6891142725944519,
      "logps/chosen": -9.789582252502441,
      "logps/rejected": -32.59271240234375,
      "loss": 0.6402732729911804,
      "memory(GiB)": 46.38,
      "nll_loss": 0.335668683052063,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15986008942127228,
      "rewards/margins": 2.269547939300537,
      "rewards/rejected": -2.1096878051757812,
      "step": 151,
      "train_speed(iter/s)": 0.005498
    },
    {
      "epoch": 0.2597042020396177,
      "grad_norm": 4.9224324226379395,
      "learning_rate": 0.00019656202114165904,
      "logits/chosen": -0.717603862285614,
      "logits/rejected": -0.7218050956726074,
      "logps/chosen": -8.468698501586914,
      "logps/rejected": -42.157318115234375,
      "loss": 0.6935219168663025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4874541759490967,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.11759095638990402,
      "rewards/margins": 2.7493767738342285,
      "rewards/rejected": -2.8669681549072266,
      "step": 152,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.26141278231619414,
      "grad_norm": 5.180264472961426,
      "learning_rate": 0.00019648812674375902,
      "logits/chosen": -0.6897304058074951,
      "logits/rejected": -0.6921338438987732,
      "logps/chosen": -6.188149929046631,
      "logps/rejected": -35.15382385253906,
      "loss": 0.6033867597579956,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2803235948085785,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18143904209136963,
      "rewards/margins": 2.3970108032226562,
      "rewards/rejected": -2.215571641921997,
      "step": 153,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.2631213625927706,
      "grad_norm": 4.898785591125488,
      "learning_rate": 0.00019641346082936872,
      "logits/chosen": -0.6851431131362915,
      "logits/rejected": -0.6863937973976135,
      "logps/chosen": -14.464362144470215,
      "logps/rejected": -23.472118377685547,
      "loss": 0.783013105392456,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4414927363395691,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.01941562071442604,
      "rewards/margins": 1.5958939790725708,
      "rewards/rejected": -1.576478362083435,
      "step": 154,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.264829942869347,
      "grad_norm": 5.930595874786377,
      "learning_rate": 0.00019633802399551473,
      "logits/chosen": -0.7089685797691345,
      "logits/rejected": -0.7100527882575989,
      "logps/chosen": -9.57446575164795,
      "logps/rejected": -26.301237106323242,
      "loss": 0.8949770927429199,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4963584840297699,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.03665952757000923,
      "rewards/margins": 1.5139617919921875,
      "rewards/rejected": -1.5506211519241333,
      "step": 155,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.2665385231459234,
      "grad_norm": 4.985401630401611,
      "learning_rate": 0.00019626181684538808,
      "logits/chosen": -0.6955317854881287,
      "logits/rejected": -0.7012707591056824,
      "logps/chosen": -6.044739723205566,
      "logps/rejected": -24.839508056640625,
      "loss": 0.9819079637527466,
      "memory(GiB)": 46.38,
      "nll_loss": 0.47508248686790466,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.06785830110311508,
      "rewards/margins": 0.7281482219696045,
      "rewards/rejected": -0.660290002822876,
      "step": 156,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.26824710342249986,
      "grad_norm": 6.6396403312683105,
      "learning_rate": 0.000196184839988339,
      "logits/chosen": -0.6605405211448669,
      "logits/rejected": -0.6651955842971802,
      "logps/chosen": -5.8500542640686035,
      "logps/rejected": -27.22307586669922,
      "loss": 0.7894259095191956,
      "memory(GiB)": 46.38,
      "nll_loss": 0.44944238662719727,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20848321914672852,
      "rewards/margins": 1.5052435398101807,
      "rewards/rejected": -1.2967603206634521,
      "step": 157,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2699556836990763,
      "grad_norm": 6.331238269805908,
      "learning_rate": 0.00019610709403987246,
      "logits/chosen": -0.6915969848632812,
      "logits/rejected": -0.6922619938850403,
      "logps/chosen": -6.449377059936523,
      "logps/rejected": -22.124189376831055,
      "loss": 0.7714009881019592,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4102553725242615,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15198560059070587,
      "rewards/margins": 1.139552116394043,
      "rewards/rejected": -0.9875664710998535,
      "step": 158,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.27166426397565274,
      "grad_norm": 4.062625408172607,
      "learning_rate": 0.00019602857962164297,
      "logits/chosen": -0.7070209383964539,
      "logits/rejected": -0.7094908952713013,
      "logps/chosen": -8.760738372802734,
      "logps/rejected": -21.83628273010254,
      "loss": 0.6808652877807617,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30313965678215027,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16821521520614624,
      "rewards/margins": 1.2700836658477783,
      "rewards/rejected": -1.1018683910369873,
      "step": 159,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.2733728442522292,
      "grad_norm": 3.5334150791168213,
      "learning_rate": 0.00019594929736144976,
      "logits/chosen": -0.6827138066291809,
      "logits/rejected": -0.6853998303413391,
      "logps/chosen": -8.673670768737793,
      "logps/rejected": -22.576623916625977,
      "loss": 0.8358533978462219,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4210669696331024,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1305871605873108,
      "rewards/margins": 1.2085453271865845,
      "rewards/rejected": -1.077958106994629,
      "step": 160,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2750814245288056,
      "grad_norm": 4.968493938446045,
      "learning_rate": 0.0001958692478932316,
      "logits/chosen": -0.6895708441734314,
      "logits/rejected": -0.6913452744483948,
      "logps/chosen": -8.595847129821777,
      "logps/rejected": -15.964265823364258,
      "loss": 0.9963687658309937,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4303150177001953,
      "rewards/accuracies": 0.59375,
      "rewards/chosen": 0.039102740585803986,
      "rewards/margins": 0.7386963367462158,
      "rewards/rejected": -0.6995935440063477,
      "step": 161,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.276790004805382,
      "grad_norm": 3.1100776195526123,
      "learning_rate": 0.00019578843185706198,
      "logits/chosen": -0.6581685543060303,
      "logits/rejected": -0.6606261730194092,
      "logps/chosen": -8.823217391967773,
      "logps/rejected": -20.12518310546875,
      "loss": 0.7960190773010254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3306158185005188,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.1894218921661377,
      "rewards/margins": 0.9922627210617065,
      "rewards/rejected": -0.8028407096862793,
      "step": 162,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.27849858508195846,
      "grad_norm": 2.9061076641082764,
      "learning_rate": 0.00019570684989914368,
      "logits/chosen": -0.7005666494369507,
      "logits/rejected": -0.7033515572547913,
      "logps/chosen": -6.614690780639648,
      "logps/rejected": -23.320758819580078,
      "loss": 0.7204984426498413,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3370620608329773,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.20666998624801636,
      "rewards/margins": 1.309868335723877,
      "rewards/rejected": -1.1031984090805054,
      "step": 163,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.2802071653585349,
      "grad_norm": 2.7546348571777344,
      "learning_rate": 0.00019562450267180396,
      "logits/chosen": -0.713110089302063,
      "logits/rejected": -0.7162569165229797,
      "logps/chosen": -11.556283950805664,
      "logps/rejected": -24.4801025390625,
      "loss": 0.600546658039093,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29867106676101685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.32357320189476013,
      "rewards/margins": 1.2928061485290527,
      "rewards/rejected": -0.969232976436615,
      "step": 164,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.28191574563511135,
      "grad_norm": 4.433908462524414,
      "learning_rate": 0.000195541390833489,
      "logits/chosen": -0.7190208435058594,
      "logits/rejected": -0.7217738628387451,
      "logps/chosen": -8.067594528198242,
      "logps/rejected": -28.02341651916504,
      "loss": 0.6388089656829834,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3460094630718231,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26204991340637207,
      "rewards/margins": 1.4149954319000244,
      "rewards/rejected": -1.152945637702942,
      "step": 165,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.28362432591168774,
      "grad_norm": 2.649824380874634,
      "learning_rate": 0.0001954575150487589,
      "logits/chosen": -0.749485969543457,
      "logits/rejected": -0.7498363852500916,
      "logps/chosen": -8.293320655822754,
      "logps/rejected": -18.8927001953125,
      "loss": 0.6769586205482483,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2774519920349121,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.14119945466518402,
      "rewards/margins": 1.063216209411621,
      "rewards/rejected": -0.9220167398452759,
      "step": 166,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2853329061882642,
      "grad_norm": 5.596877574920654,
      "learning_rate": 0.0001953728759882821,
      "logits/chosen": -0.7324296832084656,
      "logits/rejected": -0.7348483204841614,
      "logps/chosen": -10.596734046936035,
      "logps/rejected": -26.180767059326172,
      "loss": 0.6137089729309082,
      "memory(GiB)": 46.38,
      "nll_loss": 0.338160902261734,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13076083362102509,
      "rewards/margins": 1.6062695980072021,
      "rewards/rejected": -1.4755089282989502,
      "step": 167,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.2870414864648406,
      "grad_norm": 3.771359443664551,
      "learning_rate": 0.00019528747432883035,
      "logits/chosen": -0.790438711643219,
      "logits/rejected": -0.794373631477356,
      "logps/chosen": -10.444026947021484,
      "logps/rejected": -26.96920394897461,
      "loss": 0.9520989060401917,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5127608776092529,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06649212539196014,
      "rewards/margins": 1.7065045833587646,
      "rewards/rejected": -1.6400126218795776,
      "step": 168,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.28875006674141707,
      "grad_norm": 5.279207706451416,
      "learning_rate": 0.00019520131075327298,
      "logits/chosen": -0.811764657497406,
      "logits/rejected": -0.8173067569732666,
      "logps/chosen": -5.237765312194824,
      "logps/rejected": -34.11195755004883,
      "loss": 0.5845170617103577,
      "memory(GiB)": 46.38,
      "nll_loss": 0.320607453584671,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.16728658974170685,
      "rewards/margins": 2.271721363067627,
      "rewards/rejected": -2.1044344902038574,
      "step": 169,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.2904586470179935,
      "grad_norm": 3.968066453933716,
      "learning_rate": 0.00019511438595057174,
      "logits/chosen": -0.8485198020935059,
      "logits/rejected": -0.8553695678710938,
      "logps/chosen": -2.880854606628418,
      "logps/rejected": -36.118629455566406,
      "loss": 0.4797823131084442,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23597285151481628,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.19838106632232666,
      "rewards/margins": 2.7422218322753906,
      "rewards/rejected": -2.5438406467437744,
      "step": 170,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2921672272945699,
      "grad_norm": 5.908665180206299,
      "learning_rate": 0.00019502670061577498,
      "logits/chosen": -0.8754770159721375,
      "logits/rejected": -0.8793540000915527,
      "logps/chosen": -6.855566501617432,
      "logps/rejected": -41.512691497802734,
      "loss": 0.5422110557556152,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29031190276145935,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1825065314769745,
      "rewards/margins": 3.2673420906066895,
      "rewards/rejected": -3.0848352909088135,
      "step": 171,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.29387580757114634,
      "grad_norm": 9.865046501159668,
      "learning_rate": 0.00019493825545001227,
      "logits/chosen": -0.8389215469360352,
      "logits/rejected": -0.8400964736938477,
      "logps/chosen": -5.75340461730957,
      "logps/rejected": -28.154647827148438,
      "loss": 0.5484858751296997,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24028873443603516,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0924752876162529,
      "rewards/margins": 2.016855001449585,
      "rewards/rejected": -1.9243797063827515,
      "step": 172,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.2955843878477228,
      "grad_norm": 5.110999584197998,
      "learning_rate": 0.00019484905116048883,
      "logits/chosen": -0.8424903750419617,
      "logits/rejected": -0.8479667901992798,
      "logps/chosen": -10.439705848693848,
      "logps/rejected": -34.95732116699219,
      "loss": 0.9751264452934265,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6046102046966553,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.047889530658721924,
      "rewards/margins": 2.1707029342651367,
      "rewards/rejected": -2.1228134632110596,
      "step": 173,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.29729296812429923,
      "grad_norm": 5.233139514923096,
      "learning_rate": 0.00019475908846047982,
      "logits/chosen": -0.8389162421226501,
      "logits/rejected": -0.8423767685890198,
      "logps/chosen": -6.015560626983643,
      "logps/rejected": -33.967063903808594,
      "loss": 0.5880640745162964,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36994248628616333,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.29298272728919983,
      "rewards/margins": 2.3524224758148193,
      "rewards/rejected": -2.0594396591186523,
      "step": 174,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.29900154840087567,
      "grad_norm": 3.566279649734497,
      "learning_rate": 0.00019466836806932453,
      "logits/chosen": -0.8825660943984985,
      "logits/rejected": -0.8887041807174683,
      "logps/chosen": -6.019082546234131,
      "logps/rejected": -35.091529846191406,
      "loss": 0.6474583148956299,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41612136363983154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2304912507534027,
      "rewards/margins": 2.494393825531006,
      "rewards/rejected": -2.2639026641845703,
      "step": 175,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.30071012867745206,
      "grad_norm": 6.7856950759887695,
      "learning_rate": 0.00019457689071242085,
      "logits/chosen": -0.8579093813896179,
      "logits/rejected": -0.8634753227233887,
      "logps/chosen": -9.888270378112793,
      "logps/rejected": -41.228416442871094,
      "loss": 0.6879767179489136,
      "memory(GiB)": 46.38,
      "nll_loss": 0.423296183347702,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31314119696617126,
      "rewards/margins": 2.737194538116455,
      "rewards/rejected": -2.424053192138672,
      "step": 176,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3024187089540285,
      "grad_norm": 4.894569396972656,
      "learning_rate": 0.0001944846571212192,
      "logits/chosen": -0.8246264457702637,
      "logits/rejected": -0.8278872966766357,
      "logps/chosen": -9.956787109375,
      "logps/rejected": -38.914066314697266,
      "loss": 0.7229227423667908,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3963947594165802,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.03048780933022499,
      "rewards/margins": 2.575648307800293,
      "rewards/rejected": -2.5451607704162598,
      "step": 177,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.30412728923060495,
      "grad_norm": 5.153270721435547,
      "learning_rate": 0.000194391668033217,
      "logits/chosen": -0.8741589784622192,
      "logits/rejected": -0.879849374294281,
      "logps/chosen": -5.598626613616943,
      "logps/rejected": -33.6767463684082,
      "loss": 0.6376072764396667,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31503573060035706,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20653334259986877,
      "rewards/margins": 2.248800754547119,
      "rewards/rejected": -2.0422675609588623,
      "step": 178,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3058358695071814,
      "grad_norm": 6.50536584854126,
      "learning_rate": 0.00019429792419195247,
      "logits/chosen": -0.83283931016922,
      "logits/rejected": -0.8357385993003845,
      "logps/chosen": -5.801810264587402,
      "logps/rejected": -26.60141372680664,
      "loss": 0.7859883308410645,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40051573514938354,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.022562842816114426,
      "rewards/margins": 1.451593041419983,
      "rewards/rejected": -1.429030179977417,
      "step": 179,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.30754444978375783,
      "grad_norm": 2.6014249324798584,
      "learning_rate": 0.0001942034263469989,
      "logits/chosen": -0.8176060914993286,
      "logits/rejected": -0.8231288194656372,
      "logps/chosen": -8.629841804504395,
      "logps/rejected": -33.58430480957031,
      "loss": 0.6443658471107483,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34101858735084534,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10939686745405197,
      "rewards/margins": 2.1205523014068604,
      "rewards/rejected": -2.011155605316162,
      "step": 180,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3092530300603342,
      "grad_norm": 1.8803960084915161,
      "learning_rate": 0.00019410817525395855,
      "logits/chosen": -0.8076679706573486,
      "logits/rejected": -0.8171229958534241,
      "logps/chosen": -3.606616735458374,
      "logps/rejected": -41.894737243652344,
      "loss": 0.37792402505874634,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1496608406305313,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.18168124556541443,
      "rewards/margins": 2.5405263900756836,
      "rewards/rejected": -2.3588449954986572,
      "step": 181,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.31096161033691067,
      "grad_norm": 1.7322460412979126,
      "learning_rate": 0.0001940121716744566,
      "logits/chosen": -0.8273515701293945,
      "logits/rejected": -0.8358028531074524,
      "logps/chosen": -4.195955753326416,
      "logps/rejected": -42.863243103027344,
      "loss": 0.40653136372566223,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1898367702960968,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2309657633304596,
      "rewards/margins": 2.9126315116882324,
      "rewards/rejected": -2.6816658973693848,
      "step": 182,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.3126701906134871,
      "grad_norm": 7.253870010375977,
      "learning_rate": 0.00019391541637613508,
      "logits/chosen": -0.818758487701416,
      "logits/rejected": -0.8240275979042053,
      "logps/chosen": -6.234726428985596,
      "logps/rejected": -38.455230712890625,
      "loss": 0.6432409882545471,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3129611313343048,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.03605951368808746,
      "rewards/margins": 2.6540074348449707,
      "rewards/rejected": -2.617948055267334,
      "step": 183,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.31437877089006355,
      "grad_norm": 3.745917320251465,
      "learning_rate": 0.0001938179101326468,
      "logits/chosen": -0.8772048354148865,
      "logits/rejected": -0.8792482614517212,
      "logps/chosen": -10.402073860168457,
      "logps/rejected": -31.16716766357422,
      "loss": 0.7786051630973816,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4461228549480438,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.22984227538108826,
      "rewards/margins": 2.267395496368408,
      "rewards/rejected": -2.037553310394287,
      "step": 184,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.31608735116664,
      "grad_norm": 3.6105992794036865,
      "learning_rate": 0.00019371965372364915,
      "logits/chosen": -0.8841488361358643,
      "logits/rejected": -0.8877873420715332,
      "logps/chosen": -7.848278045654297,
      "logps/rejected": -28.794742584228516,
      "loss": 0.7686839699745178,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4148586094379425,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.2514728009700775,
      "rewards/margins": 2.0990750789642334,
      "rewards/rejected": -1.8476024866104126,
      "step": 185,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3177959314432164,
      "grad_norm": 4.601129531860352,
      "learning_rate": 0.00019362064793479767,
      "logits/chosen": -0.890434980392456,
      "logits/rejected": -0.8937606811523438,
      "logps/chosen": -6.078680515289307,
      "logps/rejected": -33.81269454956055,
      "loss": 0.6938453316688538,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35300731658935547,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.06243661791086197,
      "rewards/margins": 2.5140228271484375,
      "rewards/rejected": -2.4515862464904785,
      "step": 186,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3195045117197928,
      "grad_norm": 3.495896339416504,
      "learning_rate": 0.00019352089355774004,
      "logits/chosen": -0.8553616404533386,
      "logits/rejected": -0.8608652949333191,
      "logps/chosen": -5.048914432525635,
      "logps/rejected": -32.950260162353516,
      "loss": 0.5130528211593628,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27008938789367676,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2167266607284546,
      "rewards/margins": 2.171088457107544,
      "rewards/rejected": -1.9543616771697998,
      "step": 187,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.32121309199636927,
      "grad_norm": 2.0062403678894043,
      "learning_rate": 0.0001934203913901096,
      "logits/chosen": -0.8811948299407959,
      "logits/rejected": -0.8872733116149902,
      "logps/chosen": -6.987203598022461,
      "logps/rejected": -31.409507751464844,
      "loss": 0.6117798686027527,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3332153260707855,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2551400661468506,
      "rewards/margins": 2.197721004486084,
      "rewards/rejected": -1.942581057548523,
      "step": 188,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3229216722729457,
      "grad_norm": 2.5417840480804443,
      "learning_rate": 0.000193319142235519,
      "logits/chosen": -0.8587099313735962,
      "logits/rejected": -0.860413670539856,
      "logps/chosen": -10.022211074829102,
      "logps/rejected": -29.925012588500977,
      "loss": 0.6710554361343384,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4114764630794525,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2854654788970947,
      "rewards/margins": 2.179234743118286,
      "rewards/rejected": -1.893769383430481,
      "step": 189,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.32463025254952216,
      "grad_norm": 2.4238228797912598,
      "learning_rate": 0.0001932171469035538,
      "logits/chosen": -0.8661004900932312,
      "logits/rejected": -0.8710408806800842,
      "logps/chosen": -5.72200870513916,
      "logps/rejected": -31.211780548095703,
      "loss": 0.4626935124397278,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24239738285541534,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.23706884682178497,
      "rewards/margins": 2.2727468013763428,
      "rewards/rejected": -2.0356781482696533,
      "step": 190,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.32633883282609855,
      "grad_norm": 4.148837089538574,
      "learning_rate": 0.00019311440620976597,
      "logits/chosen": -0.9142186045646667,
      "logits/rejected": -0.9197511076927185,
      "logps/chosen": -6.843296051025391,
      "logps/rejected": -34.4539680480957,
      "loss": 0.5014334321022034,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3269428014755249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.398705393075943,
      "rewards/margins": 2.5132031440734863,
      "rewards/rejected": -2.1144979000091553,
      "step": 191,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.328047413102675,
      "grad_norm": 6.930333614349365,
      "learning_rate": 0.0001930109209756673,
      "logits/chosen": -0.9393826723098755,
      "logits/rejected": -0.9431489109992981,
      "logps/chosen": -12.365182876586914,
      "logps/rejected": -36.20299530029297,
      "loss": 0.9010649919509888,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5080785155296326,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.09569638967514038,
      "rewards/margins": 2.128075361251831,
      "rewards/rejected": -2.223771810531616,
      "step": 192,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.32975599337925143,
      "grad_norm": 1.8063973188400269,
      "learning_rate": 0.00019290669202872302,
      "logits/chosen": -0.9646225571632385,
      "logits/rejected": -0.9725656509399414,
      "logps/chosen": -4.754866123199463,
      "logps/rejected": -49.72484588623047,
      "loss": 0.3587495982646942,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2268112450838089,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3622514307498932,
      "rewards/margins": 4.094050884246826,
      "rewards/rejected": -3.731799602508545,
      "step": 193,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3314645736558279,
      "grad_norm": 9.937451362609863,
      "learning_rate": 0.00019280172020234496,
      "logits/chosen": -1.0251308679580688,
      "logits/rejected": -1.030665636062622,
      "logps/chosen": -10.551065444946289,
      "logps/rejected": -53.82728576660156,
      "loss": 1.1305742263793945,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5491915345191956,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.27872776985168457,
      "rewards/margins": 3.9652843475341797,
      "rewards/rejected": -4.244011878967285,
      "step": 194,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3331731539324043,
      "grad_norm": 7.0363922119140625,
      "learning_rate": 0.00019269600633588514,
      "logits/chosen": -0.98694908618927,
      "logits/rejected": -0.9900444746017456,
      "logps/chosen": -11.45719051361084,
      "logps/rejected": -51.027496337890625,
      "loss": 0.6249094009399414,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3946501910686493,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.056097134947776794,
      "rewards/margins": 3.8215420246124268,
      "rewards/rejected": -3.8776392936706543,
      "step": 195,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3348817342089807,
      "grad_norm": 7.936800003051758,
      "learning_rate": 0.0001925895512746287,
      "logits/chosen": -0.9765729904174805,
      "logits/rejected": -0.9803340435028076,
      "logps/chosen": -7.451618194580078,
      "logps/rejected": -42.50349426269531,
      "loss": 0.8021814227104187,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49508342146873474,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.0933699905872345,
      "rewards/margins": 3.030968189239502,
      "rewards/rejected": -3.124338150024414,
      "step": 196,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.33659031448555715,
      "grad_norm": 7.247754096984863,
      "learning_rate": 0.00019248235586978755,
      "logits/chosen": -0.9704450964927673,
      "logits/rejected": -0.9796913862228394,
      "logps/chosen": -5.781133651733398,
      "logps/rejected": -43.08665084838867,
      "loss": 0.5009433627128601,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3090832829475403,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17386135458946228,
      "rewards/margins": 3.0037147998809814,
      "rewards/rejected": -2.8298535346984863,
      "step": 197,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3382988947621336,
      "grad_norm": 4.252370357513428,
      "learning_rate": 0.0001923744209784933,
      "logits/chosen": -0.9834634065628052,
      "logits/rejected": -0.9903834462165833,
      "logps/chosen": -10.431640625,
      "logps/rejected": -41.34126663208008,
      "loss": 0.892498791217804,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5820133686065674,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.0897313803434372,
      "rewards/margins": 2.829003095626831,
      "rewards/rejected": -2.739271640777588,
      "step": 198,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.34000747503871004,
      "grad_norm": 7.74161958694458,
      "learning_rate": 0.00019226574746379038,
      "logits/chosen": -0.9264886975288391,
      "logits/rejected": -0.9303056597709656,
      "logps/chosen": -18.308433532714844,
      "logps/rejected": -34.25849914550781,
      "loss": 1.4104645252227783,
      "memory(GiB)": 46.38,
      "nll_loss": 1.1379212141036987,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.23661470413208008,
      "rewards/margins": 2.288109540939331,
      "rewards/rejected": -2.051494598388672,
      "step": 199,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3417160553152864,
      "grad_norm": 19.06857681274414,
      "learning_rate": 0.00019215633619462942,
      "logits/chosen": -0.9472784399986267,
      "logits/rejected": -0.9554142951965332,
      "logps/chosen": -9.5065336227417,
      "logps/rejected": -40.612945556640625,
      "loss": 1.1060166358947754,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7597510814666748,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.030025824904441833,
      "rewards/margins": 2.3827638626098633,
      "rewards/rejected": -2.35273814201355,
      "step": 200,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.34342463559186287,
      "grad_norm": 3.042719841003418,
      "learning_rate": 0.00019204618804585994,
      "logits/chosen": -0.8972377181053162,
      "logits/rejected": -0.9024269580841064,
      "logps/chosen": -6.98057222366333,
      "logps/rejected": -34.55181884765625,
      "loss": 0.5098893642425537,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2953639328479767,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.31456321477890015,
      "rewards/margins": 2.592310905456543,
      "rewards/rejected": -2.277747392654419,
      "step": 201,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3451332158684393,
      "grad_norm": 3.191251516342163,
      "learning_rate": 0.00019193530389822363,
      "logits/chosen": -0.9289453625679016,
      "logits/rejected": -0.9359845519065857,
      "logps/chosen": -9.659342765808105,
      "logps/rejected": -37.61637878417969,
      "loss": 1.0019766092300415,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6679946780204773,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1197037547826767,
      "rewards/margins": 2.4200377464294434,
      "rewards/rejected": -2.3003342151641846,
      "step": 202,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.34684179614501576,
      "grad_norm": 4.930737018585205,
      "learning_rate": 0.00019182368463834722,
      "logits/chosen": -0.8630688190460205,
      "logits/rejected": -0.8667917847633362,
      "logps/chosen": -9.344999313354492,
      "logps/rejected": -36.20460510253906,
      "loss": 0.5554157495498657,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36027616262435913,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1459665298461914,
      "rewards/margins": 2.5479657649993896,
      "rewards/rejected": -2.4019992351531982,
      "step": 203,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3485503764215922,
      "grad_norm": 2.625263214111328,
      "learning_rate": 0.00019171133115873526,
      "logits/chosen": -0.8702833652496338,
      "logits/rejected": -0.8725969195365906,
      "logps/chosen": -7.262643814086914,
      "logps/rejected": -34.7918701171875,
      "loss": 0.4979276657104492,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3470659554004669,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23693254590034485,
      "rewards/margins": 2.7716078758239746,
      "rewards/rejected": -2.534675359725952,
      "step": 204,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3502589566981686,
      "grad_norm": 4.522634029388428,
      "learning_rate": 0.0001915982443577632,
      "logits/chosen": -0.8535888195037842,
      "logits/rejected": -0.859410285949707,
      "logps/chosen": -6.171905517578125,
      "logps/rejected": -37.568458557128906,
      "loss": 0.5628905296325684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3058054447174072,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09114877134561539,
      "rewards/margins": 2.7372331619262695,
      "rewards/rejected": -2.6460840702056885,
      "step": 205,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.35196753697474503,
      "grad_norm": 3.1796517372131348,
      "learning_rate": 0.00019148442513967014,
      "logits/chosen": -0.8468549251556396,
      "logits/rejected": -0.8539971709251404,
      "logps/chosen": -9.5682954788208,
      "logps/rejected": -36.71331024169922,
      "loss": 0.6788620948791504,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4944525957107544,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10999859869480133,
      "rewards/margins": 2.5533294677734375,
      "rewards/rejected": -2.443330764770508,
      "step": 206,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3536761172513215,
      "grad_norm": 3.1952710151672363,
      "learning_rate": 0.00019136987441455144,
      "logits/chosen": -0.8667152523994446,
      "logits/rejected": -0.8705443143844604,
      "logps/chosen": -9.689000129699707,
      "logps/rejected": -30.918254852294922,
      "loss": 0.6780470013618469,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41608238220214844,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.2547697126865387,
      "rewards/margins": 2.3299858570098877,
      "rewards/rejected": -2.075216293334961,
      "step": 207,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3553846975278979,
      "grad_norm": 2.4894015789031982,
      "learning_rate": 0.00019125459309835173,
      "logits/chosen": -0.8714242577552795,
      "logits/rejected": -0.8752931356430054,
      "logps/chosen": -7.453229904174805,
      "logps/rejected": -31.325904846191406,
      "loss": 0.6907439827919006,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4350113868713379,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.17801815271377563,
      "rewards/margins": 2.264295816421509,
      "rewards/rejected": -2.086277484893799,
      "step": 208,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.35709327780447436,
      "grad_norm": 3.2602062225341797,
      "learning_rate": 0.00019113858211285725,
      "logits/chosen": -0.872046709060669,
      "logits/rejected": -0.8767064213752747,
      "logps/chosen": -4.748498439788818,
      "logps/rejected": -36.4671745300293,
      "loss": 0.47542524337768555,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24451857805252075,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20409636199474335,
      "rewards/margins": 2.852433443069458,
      "rewards/rejected": -2.6483371257781982,
      "step": 209,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.35880185808105075,
      "grad_norm": 2.723602533340454,
      "learning_rate": 0.00019102184238568878,
      "logits/chosen": -0.872526228427887,
      "logits/rejected": -0.8720419406890869,
      "logps/chosen": -9.495769500732422,
      "logps/rejected": -35.4581184387207,
      "loss": 0.6998460292816162,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4141753911972046,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.08976475894451141,
      "rewards/margins": 2.3428900241851807,
      "rewards/rejected": -2.2531251907348633,
      "step": 210,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3605104383576272,
      "grad_norm": 7.090069770812988,
      "learning_rate": 0.00019090437485029413,
      "logits/chosen": -0.8751546740531921,
      "logits/rejected": -0.8804129362106323,
      "logps/chosen": -5.539629936218262,
      "logps/rejected": -41.93748092651367,
      "loss": 0.414511114358902,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24404364824295044,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2657608985900879,
      "rewards/margins": 3.283001184463501,
      "rewards/rejected": -3.017240524291992,
      "step": 211,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.36221901863420364,
      "grad_norm": 3.3576340675354004,
      "learning_rate": 0.00019078618044594052,
      "logits/chosen": -0.9006087779998779,
      "logits/rejected": -0.9030022621154785,
      "logps/chosen": -7.570390224456787,
      "logps/rejected": -34.19873046875,
      "loss": 0.4811999499797821,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29778629541397095,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24354203045368195,
      "rewards/margins": 2.750795364379883,
      "rewards/rejected": -2.5072531700134277,
      "step": 212,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3639275989107801,
      "grad_norm": 4.36439323425293,
      "learning_rate": 0.00019066726011770726,
      "logits/chosen": -0.8826490640640259,
      "logits/rejected": -0.8854631185531616,
      "logps/chosen": -7.964271068572998,
      "logps/rejected": -36.53171920776367,
      "loss": 0.7800939679145813,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4539424479007721,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1163502037525177,
      "rewards/margins": 2.651106357574463,
      "rewards/rejected": -2.5347559452056885,
      "step": 213,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3656361791873565,
      "grad_norm": 3.075863838195801,
      "learning_rate": 0.00019054761481647817,
      "logits/chosen": -0.9236173033714294,
      "logits/rejected": -0.9311164021492004,
      "logps/chosen": -5.548548698425293,
      "logps/rejected": -39.62729263305664,
      "loss": 0.5424690842628479,
      "memory(GiB)": 46.38,
      "nll_loss": 0.37887126207351685,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2379705011844635,
      "rewards/margins": 2.8928284645080566,
      "rewards/rejected": -2.654857873916626,
      "step": 214,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3673447594639329,
      "grad_norm": 4.081761360168457,
      "learning_rate": 0.0001904272454989338,
      "logits/chosen": -0.9173122644424438,
      "logits/rejected": -0.9237719774246216,
      "logps/chosen": -7.586677551269531,
      "logps/rejected": -39.44381332397461,
      "loss": 0.7125780582427979,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4088558554649353,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10065207630395889,
      "rewards/margins": 2.89681077003479,
      "rewards/rejected": -2.796158790588379,
      "step": 215,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.36905333974050936,
      "grad_norm": 6.38205099105835,
      "learning_rate": 0.00019030615312754412,
      "logits/chosen": -0.8755264282226562,
      "logits/rejected": -0.8772911429405212,
      "logps/chosen": -10.058317184448242,
      "logps/rejected": -29.399351119995117,
      "loss": 0.6085036993026733,
      "memory(GiB)": 46.38,
      "nll_loss": 0.37356507778167725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.257190465927124,
      "rewards/margins": 2.3912007808685303,
      "rewards/rejected": -2.1340105533599854,
      "step": 216,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3707619200170858,
      "grad_norm": 3.965923547744751,
      "learning_rate": 0.00019018433867056043,
      "logits/chosen": -0.8582707047462463,
      "logits/rejected": -0.8643903136253357,
      "logps/chosen": -7.274492263793945,
      "logps/rejected": -38.09757995605469,
      "loss": 0.7568970918655396,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5492669343948364,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.09097186475992203,
      "rewards/margins": 2.650477170944214,
      "rewards/rejected": -2.5595054626464844,
      "step": 217,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.37247050029366224,
      "grad_norm": 6.908701419830322,
      "learning_rate": 0.00019006180310200788,
      "logits/chosen": -0.8987607955932617,
      "logits/rejected": -0.9025367498397827,
      "logps/chosen": -9.353863716125488,
      "logps/rejected": -41.96833801269531,
      "loss": 0.5579907298088074,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32021838426589966,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.06166917830705643,
      "rewards/margins": 3.1308600902557373,
      "rewards/rejected": -3.069190740585327,
      "step": 218,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3741790805702387,
      "grad_norm": 11.417703628540039,
      "learning_rate": 0.00018993854740167768,
      "logits/chosen": -0.8815793991088867,
      "logits/rejected": -0.8841702342033386,
      "logps/chosen": -8.630674362182617,
      "logps/rejected": -32.739219665527344,
      "loss": 0.7540084719657898,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42135101556777954,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.017735840752720833,
      "rewards/margins": 2.403879165649414,
      "rewards/rejected": -2.386143445968628,
      "step": 219,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3758876608468151,
      "grad_norm": 7.359007358551025,
      "learning_rate": 0.00018981457255511902,
      "logits/chosen": -0.8066080212593079,
      "logits/rejected": -0.8113219141960144,
      "logps/chosen": -10.087733268737793,
      "logps/rejected": -34.230953216552734,
      "loss": 0.7611066102981567,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38534992933273315,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.18794448673725128,
      "rewards/margins": 2.276632785797119,
      "rewards/rejected": -2.088688373565674,
      "step": 220,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3775962411233915,
      "grad_norm": 2.5772690773010254,
      "learning_rate": 0.00018968987955363153,
      "logits/chosen": -0.7865528464317322,
      "logits/rejected": -0.7891740202903748,
      "logps/chosen": -13.03406810760498,
      "logps/rejected": -25.763931274414062,
      "loss": 0.8478187918663025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5038392543792725,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.3180733621120453,
      "rewards/margins": 1.6984466314315796,
      "rewards/rejected": -1.3803733587265015,
      "step": 221,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.37930482139996796,
      "grad_norm": 3.7778072357177734,
      "learning_rate": 0.00018956446939425716,
      "logits/chosen": -0.826096773147583,
      "logits/rejected": -0.8322123885154724,
      "logps/chosen": -9.263298988342285,
      "logps/rejected": -25.442668914794922,
      "loss": 0.6595274209976196,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3389233946800232,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24548465013504028,
      "rewards/margins": 1.5120353698730469,
      "rewards/rejected": -1.2665507793426514,
      "step": 222,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3810134016765444,
      "grad_norm": 4.857659339904785,
      "learning_rate": 0.00018943834307977216,
      "logits/chosen": -0.8075899481773376,
      "logits/rejected": -0.8120123744010925,
      "logps/chosen": -4.7845778465271,
      "logps/rejected": -21.904151916503906,
      "loss": 0.6341399550437927,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2501222789287567,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19457918405532837,
      "rewards/margins": 1.1304917335510254,
      "rewards/rejected": -0.9359126091003418,
      "step": 223,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.38272198195312085,
      "grad_norm": 2.3448665142059326,
      "learning_rate": 0.00018931150161867916,
      "logits/chosen": -0.882806658744812,
      "logits/rejected": -0.8879884481430054,
      "logps/chosen": -9.03802490234375,
      "logps/rejected": -24.53693199157715,
      "loss": 0.7897974848747253,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38777855038642883,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.1946476548910141,
      "rewards/margins": 1.5893993377685547,
      "rewards/rejected": -1.3947515487670898,
      "step": 224,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.38443056222969724,
      "grad_norm": 11.419641494750977,
      "learning_rate": 0.00018918394602519914,
      "logits/chosen": -0.8994475603103638,
      "logits/rejected": -0.9015949964523315,
      "logps/chosen": -7.98261022567749,
      "logps/rejected": -27.965211868286133,
      "loss": 0.7503970265388489,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4178537130355835,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.22919557988643646,
      "rewards/margins": 1.8219597339630127,
      "rewards/rejected": -1.592764139175415,
      "step": 225,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.3861391425062737,
      "grad_norm": 5.716240406036377,
      "learning_rate": 0.00018905567731926324,
      "logits/chosen": -0.8648837208747864,
      "logits/rejected": -0.8741611838340759,
      "logps/chosen": -6.251923084259033,
      "logps/rejected": -36.683223724365234,
      "loss": 0.5255903601646423,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3399491310119629,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2363051176071167,
      "rewards/margins": 2.3525753021240234,
      "rewards/rejected": -2.1162703037261963,
      "step": 226,
      "train_speed(iter/s)": 0.005497
    },
    {
      "epoch": 0.3878477227828501,
      "grad_norm": 2.4492273330688477,
      "learning_rate": 0.00018892669652650456,
      "logits/chosen": -0.8707509636878967,
      "logits/rejected": -0.8740925192832947,
      "logps/chosen": -8.421378135681152,
      "logps/rejected": -34.801605224609375,
      "loss": 0.6341971158981323,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4148554801940918,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3119761645793915,
      "rewards/margins": 2.5685765743255615,
      "rewards/rejected": -2.2566003799438477,
      "step": 227,
      "train_speed(iter/s)": 0.005496
    },
    {
      "epoch": 0.38955630305942657,
      "grad_norm": 7.605607986450195,
      "learning_rate": 0.00018879700467825013,
      "logits/chosen": -0.8900420069694519,
      "logits/rejected": -0.8933430910110474,
      "logps/chosen": -5.360377311706543,
      "logps/rejected": -33.13306427001953,
      "loss": 0.4900038540363312,
      "memory(GiB)": 46.38,
      "nll_loss": 0.28295254707336426,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3359947204589844,
      "rewards/margins": 2.5908119678497314,
      "rewards/rejected": -2.254817008972168,
      "step": 228,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.391264883336003,
      "grad_norm": 8.061660766601562,
      "learning_rate": 0.00018866660281151247,
      "logits/chosen": -0.9369041919708252,
      "logits/rejected": -0.9415433406829834,
      "logps/chosen": -4.601605415344238,
      "logps/rejected": -35.486732482910156,
      "loss": 0.5184635519981384,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2989267408847809,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22662585973739624,
      "rewards/margins": 2.7763917446136475,
      "rewards/rejected": -2.5497660636901855,
      "step": 229,
      "train_speed(iter/s)": 0.005495
    },
    {
      "epoch": 0.3929734636125794,
      "grad_norm": 3.7317209243774414,
      "learning_rate": 0.0001885354919689815,
      "logits/chosen": -0.9507468342781067,
      "logits/rejected": -0.9519726634025574,
      "logps/chosen": -6.97619104385376,
      "logps/rejected": -38.649261474609375,
      "loss": 0.4864623546600342,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2420811951160431,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16122400760650635,
      "rewards/margins": 2.8739140033721924,
      "rewards/rejected": -2.7126896381378174,
      "step": 230,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.39468204388915584,
      "grad_norm": 2.446930408477783,
      "learning_rate": 0.00018840367319901592,
      "logits/chosen": -0.9463286399841309,
      "logits/rejected": -0.9530838131904602,
      "logps/chosen": -4.321158409118652,
      "logps/rejected": -45.260074615478516,
      "loss": 0.2898697853088379,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19011972844600677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2383752316236496,
      "rewards/margins": 3.6623451709747314,
      "rewards/rejected": -3.4239697456359863,
      "step": 231,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.3963906241657323,
      "grad_norm": 10.94664478302002,
      "learning_rate": 0.0001882711475556352,
      "logits/chosen": -0.9894860982894897,
      "logits/rejected": -0.9986059665679932,
      "logps/chosen": -7.955844879150391,
      "logps/rejected": -57.02360534667969,
      "loss": 1.0545183420181274,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7007216215133667,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.0017464309930801392,
      "rewards/margins": 4.065786361694336,
      "rewards/rejected": -4.067533016204834,
      "step": 232,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.39809920444230873,
      "grad_norm": 5.375171661376953,
      "learning_rate": 0.0001881379160985108,
      "logits/chosen": -1.0208549499511719,
      "logits/rejected": -1.0248205661773682,
      "logps/chosen": -7.056022644042969,
      "logps/rejected": -52.94482421875,
      "loss": 0.6524523496627808,
      "memory(GiB)": 46.38,
      "nll_loss": 0.44853687286376953,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08947555720806122,
      "rewards/margins": 4.471720218658447,
      "rewards/rejected": -4.382244110107422,
      "step": 233,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.3998077847188852,
      "grad_norm": 12.273642539978027,
      "learning_rate": 0.00018800397989295794,
      "logits/chosen": -0.9972729682922363,
      "logits/rejected": -1.0022565126419067,
      "logps/chosen": -10.15302848815918,
      "logps/rejected": -46.114967346191406,
      "loss": 0.7069646716117859,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4566623270511627,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1436801254749298,
      "rewards/margins": 3.605038642883301,
      "rewards/rejected": -3.4613585472106934,
      "step": 234,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.40151636499546156,
      "grad_norm": 8.246232986450195,
      "learning_rate": 0.00018786934000992688,
      "logits/chosen": -1.0225094556808472,
      "logits/rejected": -1.019821286201477,
      "logps/chosen": -11.058813095092773,
      "logps/rejected": -53.49971008300781,
      "loss": 0.811690092086792,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42450016736984253,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.12926799058914185,
      "rewards/margins": 4.236933708190918,
      "rewards/rejected": -4.366201877593994,
      "step": 235,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.403224945272038,
      "grad_norm": 13.076598167419434,
      "learning_rate": 0.00018773399752599463,
      "logits/chosen": -1.001189112663269,
      "logits/rejected": -1.0065340995788574,
      "logps/chosen": -8.285541534423828,
      "logps/rejected": -48.76884460449219,
      "loss": 0.6642578840255737,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3762781023979187,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.013312354683876038,
      "rewards/margins": 3.655507802963257,
      "rewards/rejected": -3.642195701599121,
      "step": 236,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.40493352554861445,
      "grad_norm": 3.6171457767486572,
      "learning_rate": 0.000187597953523356,
      "logits/chosen": -0.9681710600852966,
      "logits/rejected": -0.9759187698364258,
      "logps/chosen": -8.311320304870605,
      "logps/rejected": -54.762943267822266,
      "loss": 0.5605093240737915,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39821937680244446,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.021861745044589043,
      "rewards/margins": 4.216716289520264,
      "rewards/rejected": -4.194854736328125,
      "step": 237,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4066421058251909,
      "grad_norm": 15.052586555480957,
      "learning_rate": 0.00018746120908981527,
      "logits/chosen": -0.9332698583602905,
      "logits/rejected": -0.9356809258460999,
      "logps/chosen": -6.916642189025879,
      "logps/rejected": -39.75300979614258,
      "loss": 0.5308848023414612,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33774426579475403,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11101798713207245,
      "rewards/margins": 3.2900469303131104,
      "rewards/rejected": -3.1790287494659424,
      "step": 238,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.40835068610176734,
      "grad_norm": 1.4588333368301392,
      "learning_rate": 0.00018732376531877725,
      "logits/chosen": -0.8663907647132874,
      "logits/rejected": -0.8769683241844177,
      "logps/chosen": -7.994008541107178,
      "logps/rejected": -53.477638244628906,
      "loss": 0.3602023124694824,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27253228425979614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.39557039737701416,
      "rewards/margins": 4.195072174072266,
      "rewards/rejected": -3.799501895904541,
      "step": 239,
      "train_speed(iter/s)": 0.005494
    },
    {
      "epoch": 0.4100592663783437,
      "grad_norm": 2.3394670486450195,
      "learning_rate": 0.0001871856233092387,
      "logits/chosen": -0.8316817283630371,
      "logits/rejected": -0.8333123326301575,
      "logps/chosen": -9.785093307495117,
      "logps/rejected": -40.74665069580078,
      "loss": 0.4092681407928467,
      "memory(GiB)": 46.38,
      "nll_loss": 0.260936975479126,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18386238813400269,
      "rewards/margins": 2.943046808242798,
      "rewards/rejected": -2.7591843605041504,
      "step": 240,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.41176784665492017,
      "grad_norm": 15.97940731048584,
      "learning_rate": 0.00018704678416577941,
      "logits/chosen": -0.8120774030685425,
      "logits/rejected": -0.8134472370147705,
      "logps/chosen": -11.521053314208984,
      "logps/rejected": -35.660865783691406,
      "loss": 0.8933948874473572,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5571330785751343,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.0759764090180397,
      "rewards/margins": 2.2313168048858643,
      "rewards/rejected": -2.30729341506958,
      "step": 241,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.4134764269314966,
      "grad_norm": 4.722672462463379,
      "learning_rate": 0.00018690724899855348,
      "logits/chosen": -0.7959482669830322,
      "logits/rejected": -0.7984044551849365,
      "logps/chosen": -5.855531692504883,
      "logps/rejected": -40.35407257080078,
      "loss": 0.49105608463287354,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2971791923046112,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.221318781375885,
      "rewards/margins": 3.0078468322753906,
      "rewards/rejected": -2.7865281105041504,
      "step": 242,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.41518500720807305,
      "grad_norm": 6.576427936553955,
      "learning_rate": 0.00018676701892328046,
      "logits/chosen": -0.7806111574172974,
      "logits/rejected": -0.783092200756073,
      "logps/chosen": -4.328828811645508,
      "logps/rejected": -40.24433517456055,
      "loss": 0.45146334171295166,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30311349034309387,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22997835278511047,
      "rewards/margins": 3.151667833328247,
      "rewards/rejected": -2.921689510345459,
      "step": 243,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4168935874846495,
      "grad_norm": 3.0237584114074707,
      "learning_rate": 0.0001866260950612362,
      "logits/chosen": -0.768136203289032,
      "logits/rejected": -0.7717478275299072,
      "logps/chosen": -4.951301097869873,
      "logps/rejected": -33.46363830566406,
      "loss": 0.4757350981235504,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20984753966331482,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.28256160020828247,
      "rewards/margins": 2.4234559535980225,
      "rewards/rejected": -2.140894651412964,
      "step": 244,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4186021677612259,
      "grad_norm": 6.6580657958984375,
      "learning_rate": 0.0001864844785392442,
      "logits/chosen": -0.745742917060852,
      "logits/rejected": -0.7464924454689026,
      "logps/chosen": -10.95466136932373,
      "logps/rejected": -44.5806999206543,
      "loss": 0.5750457048416138,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3861500322818756,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.1691206991672516,
      "rewards/margins": 3.562136650085449,
      "rewards/rejected": -3.3930160999298096,
      "step": 245,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.42031074803780233,
      "grad_norm": 6.422461032867432,
      "learning_rate": 0.00018634217048966637,
      "logits/chosen": -0.7587261199951172,
      "logits/rejected": -0.7620642185211182,
      "logps/chosen": -6.3410139083862305,
      "logps/rejected": -33.94182586669922,
      "loss": 0.6545197367668152,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30830273032188416,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.06438297033309937,
      "rewards/margins": 2.2670321464538574,
      "rewards/rejected": -2.3314151763916016,
      "step": 246,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.42201932831437877,
      "grad_norm": 4.497966289520264,
      "learning_rate": 0.00018619917205039407,
      "logits/chosen": -0.7642217874526978,
      "logits/rejected": -0.7698625326156616,
      "logps/chosen": -6.284250259399414,
      "logps/rejected": -46.83761215209961,
      "loss": 0.4105835258960724,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2564704418182373,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22518175840377808,
      "rewards/margins": 3.5758605003356934,
      "rewards/rejected": -3.3506786823272705,
      "step": 247,
      "train_speed(iter/s)": 0.005493
    },
    {
      "epoch": 0.4237279085909552,
      "grad_norm": 3.6431024074554443,
      "learning_rate": 0.00018605548436483906,
      "logits/chosen": -0.7721464037895203,
      "logits/rejected": -0.7743059396743774,
      "logps/chosen": -8.004281044006348,
      "logps/rejected": -34.3867073059082,
      "loss": 0.6020833849906921,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3395420014858246,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2702520787715912,
      "rewards/margins": 2.563030242919922,
      "rewards/rejected": -2.292778491973877,
      "step": 248,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.42543648886753166,
      "grad_norm": 2.3539841175079346,
      "learning_rate": 0.0001859111085819242,
      "logits/chosen": -0.7793521285057068,
      "logits/rejected": -0.7824581265449524,
      "logps/chosen": -12.44356918334961,
      "logps/rejected": -37.551902770996094,
      "loss": 0.569586455821991,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4044492542743683,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30807700753211975,
      "rewards/margins": 2.927927017211914,
      "rewards/rejected": -2.6198501586914062,
      "step": 249,
      "train_speed(iter/s)": 0.005492
    },
    {
      "epoch": 0.42714506914410805,
      "grad_norm": 3.378519058227539,
      "learning_rate": 0.00018576604585607441,
      "logits/chosen": -0.7945507168769836,
      "logits/rejected": -0.7992746233940125,
      "logps/chosen": -7.014500141143799,
      "logps/rejected": -46.093238830566406,
      "loss": 0.47747644782066345,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2733898162841797,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10495103895664215,
      "rewards/margins": 3.2655720710754395,
      "rewards/rejected": -3.160621166229248,
      "step": 250,
      "train_speed(iter/s)": 0.005491
    },
    {
      "epoch": 0.4288536494206845,
      "grad_norm": 4.231056213378906,
      "learning_rate": 0.00018562029734720743,
      "logits/chosen": -0.8051905632019043,
      "logits/rejected": -0.8092507123947144,
      "logps/chosen": -6.725405693054199,
      "logps/rejected": -37.768829345703125,
      "loss": 0.5110489726066589,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31649249792099,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19485878944396973,
      "rewards/margins": 2.8023769855499268,
      "rewards/rejected": -2.607518196105957,
      "step": 251,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.43056222969726093,
      "grad_norm": 11.981834411621094,
      "learning_rate": 0.00018547386422072438,
      "logits/chosen": -0.7716567516326904,
      "logits/rejected": -0.773201584815979,
      "logps/chosen": -9.067695617675781,
      "logps/rejected": -36.645240783691406,
      "loss": 0.8504749536514282,
      "memory(GiB)": 46.38,
      "nll_loss": 0.514085054397583,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.07366381585597992,
      "rewards/margins": 2.7288200855255127,
      "rewards/rejected": -2.8024840354919434,
      "step": 252,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4322708099738374,
      "grad_norm": 8.43531608581543,
      "learning_rate": 0.0001853267476475007,
      "logits/chosen": -0.8169345855712891,
      "logits/rejected": -0.8159929513931274,
      "logps/chosen": -11.076251029968262,
      "logps/rejected": -42.98237991333008,
      "loss": 0.7847392559051514,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5229505300521851,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1810646504163742,
      "rewards/margins": 3.4308125972747803,
      "rewards/rejected": -3.2497482299804688,
      "step": 253,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4339793902504138,
      "grad_norm": 6.133362770080566,
      "learning_rate": 0.0001851789488038765,
      "logits/chosen": -0.7999134063720703,
      "logits/rejected": -0.8003981709480286,
      "logps/chosen": -6.108333587646484,
      "logps/rejected": -41.47050857543945,
      "loss": 0.6103618741035461,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3681967258453369,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1454550325870514,
      "rewards/margins": 3.3938965797424316,
      "rewards/rejected": -3.248441457748413,
      "step": 254,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4356879705269902,
      "grad_norm": 27.34607696533203,
      "learning_rate": 0.0001850304688716475,
      "logits/chosen": -0.7655246257781982,
      "logits/rejected": -0.7673658132553101,
      "logps/chosen": -9.124423027038574,
      "logps/rejected": -40.40342330932617,
      "loss": 0.9848135709762573,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6617517471313477,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10712965577840805,
      "rewards/margins": 2.936744213104248,
      "rewards/rejected": -2.8296148777008057,
      "step": 255,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.43739655080356665,
      "grad_norm": 3.467927932739258,
      "learning_rate": 0.00018488130903805523,
      "logits/chosen": -0.7815067768096924,
      "logits/rejected": -0.7830615639686584,
      "logps/chosen": -6.11367654800415,
      "logps/rejected": -43.792381286621094,
      "loss": 0.3926101326942444,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2346831113100052,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3161233365535736,
      "rewards/margins": 3.5978031158447266,
      "rewards/rejected": -3.281679630279541,
      "step": 256,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4391051310801431,
      "grad_norm": 3.6716325283050537,
      "learning_rate": 0.00018473147049577774,
      "logits/chosen": -0.8119379878044128,
      "logits/rejected": -0.8167931437492371,
      "logps/chosen": -5.306201934814453,
      "logps/rejected": -48.548152923583984,
      "loss": 0.3491962254047394,
      "memory(GiB)": 46.38,
      "nll_loss": 0.214007169008255,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19937750697135925,
      "rewards/margins": 3.6415648460388184,
      "rewards/rejected": -3.442187547683716,
      "step": 257,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.44081371135671954,
      "grad_norm": 25.686939239501953,
      "learning_rate": 0.00018458095444292004,
      "logits/chosen": -0.825552761554718,
      "logits/rejected": -0.8325546383857727,
      "logps/chosen": -5.261894702911377,
      "logps/rejected": -62.226497650146484,
      "loss": 0.434018075466156,
      "memory(GiB)": 46.38,
      "nll_loss": 0.287321001291275,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.32799047231674194,
      "rewards/margins": 4.790035247802734,
      "rewards/rejected": -4.462045192718506,
      "step": 258,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.442522291633296,
      "grad_norm": 6.754378318786621,
      "learning_rate": 0.00018442976208300448,
      "logits/chosen": -0.8270843029022217,
      "logits/rejected": -0.8281224966049194,
      "logps/chosen": -8.492132186889648,
      "logps/rejected": -36.23853302001953,
      "loss": 0.6711763739585876,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3068382143974304,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05488736927509308,
      "rewards/margins": 2.777843952178955,
      "rewards/rejected": -2.722956418991089,
      "step": 259,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.44423087190987237,
      "grad_norm": 4.042886257171631,
      "learning_rate": 0.00018427789462496113,
      "logits/chosen": -0.8525445461273193,
      "logits/rejected": -0.8591835498809814,
      "logps/chosen": -8.231825828552246,
      "logps/rejected": -47.20307540893555,
      "loss": 0.8233700394630432,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5154776573181152,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.00042735040187835693,
      "rewards/margins": 3.4719269275665283,
      "rewards/rejected": -3.4723546504974365,
      "step": 260,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4459394521864488,
      "grad_norm": 12.385782241821289,
      "learning_rate": 0.00018412535328311814,
      "logits/chosen": -0.822756290435791,
      "logits/rejected": -0.8276916742324829,
      "logps/chosen": -7.529094219207764,
      "logps/rejected": -39.26826477050781,
      "loss": 0.7842540740966797,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41948723793029785,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.09604156017303467,
      "rewards/margins": 2.7830724716186523,
      "rewards/rejected": -2.8791141510009766,
      "step": 261,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.44764803246302526,
      "grad_norm": 2.720581531524658,
      "learning_rate": 0.00018397213927719199,
      "logits/chosen": -0.7854418158531189,
      "logits/rejected": -0.7872219085693359,
      "logps/chosen": -7.754611968994141,
      "logps/rejected": -47.40762710571289,
      "loss": 0.36977341771125793,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23527176678180695,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1266416311264038,
      "rewards/margins": 3.6666605472564697,
      "rewards/rejected": -3.5400190353393555,
      "step": 262,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4493566127396017,
      "grad_norm": 3.862173557281494,
      "learning_rate": 0.00018381825383227782,
      "logits/chosen": -0.8271799087524414,
      "logits/rejected": -0.831858217716217,
      "logps/chosen": -6.855618476867676,
      "logps/rejected": -46.51610565185547,
      "loss": 0.4688463509082794,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3772401511669159,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2653267979621887,
      "rewards/margins": 3.657702922821045,
      "rewards/rejected": -3.392376184463501,
      "step": 263,
      "train_speed(iter/s)": 0.00549
    },
    {
      "epoch": 0.45106519301617815,
      "grad_norm": 2.1108744144439697,
      "learning_rate": 0.00018366369817883954,
      "logits/chosen": -0.8001513481140137,
      "logits/rejected": -0.8060557842254639,
      "logps/chosen": -5.957093715667725,
      "logps/rejected": -49.00318908691406,
      "loss": 0.42761629819869995,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2863171100616455,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.46388381719589233,
      "rewards/margins": 3.825634002685547,
      "rewards/rejected": -3.3617501258850098,
      "step": 264,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.45277377329275453,
      "grad_norm": 4.1208696365356445,
      "learning_rate": 0.00018350847355270004,
      "logits/chosen": -0.7904451489448547,
      "logits/rejected": -0.7935886979103088,
      "logps/chosen": -12.136712074279785,
      "logps/rejected": -48.772483825683594,
      "loss": 0.6188664436340332,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4114742577075958,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2880480885505676,
      "rewards/margins": 3.680865526199341,
      "rewards/rejected": -3.392817258834839,
      "step": 265,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.454482353569331,
      "grad_norm": 5.501693248748779,
      "learning_rate": 0.00018335258119503127,
      "logits/chosen": -0.8123242259025574,
      "logits/rejected": -0.8159137964248657,
      "logps/chosen": -6.165188789367676,
      "logps/rejected": -47.05010986328125,
      "loss": 0.45226797461509705,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35374370217323303,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20915599167346954,
      "rewards/margins": 3.658740997314453,
      "rewards/rejected": -3.4495849609375,
      "step": 266,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4561909338459074,
      "grad_norm": 11.674309730529785,
      "learning_rate": 0.00018319602235234442,
      "logits/chosen": -0.7954471111297607,
      "logits/rejected": -0.7979443073272705,
      "logps/chosen": -9.799958229064941,
      "logps/rejected": -51.7642822265625,
      "loss": 0.7218329310417175,
      "memory(GiB)": 46.38,
      "nll_loss": 0.44642627239227295,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.12691040337085724,
      "rewards/margins": 3.8264598846435547,
      "rewards/rejected": -3.9533700942993164,
      "step": 267,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.45789951412248386,
      "grad_norm": 21.236757278442383,
      "learning_rate": 0.00018303879827647975,
      "logits/chosen": -0.7900252938270569,
      "logits/rejected": -0.7937359809875488,
      "logps/chosen": -9.592187881469727,
      "logps/rejected": -43.07557678222656,
      "loss": 0.6332522034645081,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5019268989562988,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1493360996246338,
      "rewards/margins": 3.395822763442993,
      "rewards/rejected": -3.2464866638183594,
      "step": 268,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4596080943990603,
      "grad_norm": 13.447307586669922,
      "learning_rate": 0.00018288091022459685,
      "logits/chosen": -0.7903649210929871,
      "logits/rejected": -0.792717695236206,
      "logps/chosen": -8.880897521972656,
      "logps/rejected": -48.42265701293945,
      "loss": 1.129546046257019,
      "memory(GiB)": 46.38,
      "nll_loss": 0.9457821249961853,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.0744566023349762,
      "rewards/margins": 3.6476986408233643,
      "rewards/rejected": -3.7221555709838867,
      "step": 269,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4613166746756367,
      "grad_norm": 15.565564155578613,
      "learning_rate": 0.00018272235945916436,
      "logits/chosen": -0.7472554445266724,
      "logits/rejected": -0.7549644708633423,
      "logps/chosen": -13.44924259185791,
      "logps/rejected": -50.098731994628906,
      "loss": 1.1531708240509033,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8039681911468506,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.14791038632392883,
      "rewards/margins": 3.323791980743408,
      "rewards/rejected": -3.4717020988464355,
      "step": 270,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.46302525495221314,
      "grad_norm": 12.007596015930176,
      "learning_rate": 0.00018256314724794992,
      "logits/chosen": -0.8131598234176636,
      "logits/rejected": -0.8223381638526917,
      "logps/chosen": -5.585975646972656,
      "logps/rejected": -48.965389251708984,
      "loss": 0.4150363802909851,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2882520854473114,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21873319149017334,
      "rewards/margins": 3.403482675552368,
      "rewards/rejected": -3.1847496032714844,
      "step": 271,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4647338352287896,
      "grad_norm": 16.59933853149414,
      "learning_rate": 0.00018240327486401022,
      "logits/chosen": -0.8030964136123657,
      "logits/rejected": -0.8079983592033386,
      "logps/chosen": -8.950094223022461,
      "logps/rejected": -47.7308349609375,
      "loss": 0.9466665387153625,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6910676956176758,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.06938548386096954,
      "rewards/margins": 3.2243666648864746,
      "rewards/rejected": -3.293752431869507,
      "step": 272,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.466442415505366,
      "grad_norm": 4.702775955200195,
      "learning_rate": 0.00018224274358568052,
      "logits/chosen": -0.8188191652297974,
      "logits/rejected": -0.8212082982063293,
      "logps/chosen": -7.591650485992432,
      "logps/rejected": -37.239356994628906,
      "loss": 0.5474280714988708,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3606598973274231,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.29892435669898987,
      "rewards/margins": 3.0769505500793457,
      "rewards/rejected": -2.7780261039733887,
      "step": 273,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.46815099578194247,
      "grad_norm": 27.876619338989258,
      "learning_rate": 0.00018208155469656462,
      "logits/chosen": -0.8266133666038513,
      "logits/rejected": -0.8309741020202637,
      "logps/chosen": -6.996228218078613,
      "logps/rejected": -38.674766540527344,
      "loss": 0.8267521858215332,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5320640206336975,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.10675782710313797,
      "rewards/margins": 2.87463116645813,
      "rewards/rejected": -2.7678730487823486,
      "step": 274,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.46985957605851886,
      "grad_norm": 7.991383075714111,
      "learning_rate": 0.00018191970948552464,
      "logits/chosen": -0.8153300285339355,
      "logits/rejected": -0.8158446550369263,
      "logps/chosen": -8.539220809936523,
      "logps/rejected": -34.5740966796875,
      "loss": 0.5116520524024963,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2977459728717804,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13776344060897827,
      "rewards/margins": 2.662919282913208,
      "rewards/rejected": -2.525156021118164,
      "step": 275,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4715681563350953,
      "grad_norm": 4.298219203948975,
      "learning_rate": 0.00018175720924667056,
      "logits/chosen": -0.853663444519043,
      "logits/rejected": -0.8555605411529541,
      "logps/chosen": -7.882385730743408,
      "logps/rejected": -27.5541934967041,
      "loss": 0.5915999412536621,
      "memory(GiB)": 46.38,
      "nll_loss": 0.284732460975647,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.12615913152694702,
      "rewards/margins": 1.954941749572754,
      "rewards/rejected": -1.8287824392318726,
      "step": 276,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.47327673661167174,
      "grad_norm": 3.9946365356445312,
      "learning_rate": 0.00018159405527934996,
      "logits/chosen": -0.8487964868545532,
      "logits/rejected": -0.8562285304069519,
      "logps/chosen": -7.136427879333496,
      "logps/rejected": -37.46531677246094,
      "loss": 0.5510011315345764,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2794538140296936,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.141952246427536,
      "rewards/margins": 2.4826676845550537,
      "rewards/rejected": -2.3407156467437744,
      "step": 277,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4749853168882482,
      "grad_norm": 1.9724056720733643,
      "learning_rate": 0.0001814302488881376,
      "logits/chosen": -0.8201677799224854,
      "logits/rejected": -0.8276962041854858,
      "logps/chosen": -4.926413536071777,
      "logps/rejected": -38.361087799072266,
      "loss": 0.45584461092948914,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2575777769088745,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.190385103225708,
      "rewards/margins": 2.507413387298584,
      "rewards/rejected": -2.317028522491455,
      "step": 278,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4766938971648246,
      "grad_norm": 8.133130073547363,
      "learning_rate": 0.00018126579138282503,
      "logits/chosen": -0.8125935792922974,
      "logits/rejected": -0.8118517398834229,
      "logps/chosen": -9.689276695251465,
      "logps/rejected": -29.489994049072266,
      "loss": 0.7013036608695984,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3926777243614197,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06979987025260925,
      "rewards/margins": 2.248946189880371,
      "rewards/rejected": -2.1791462898254395,
      "step": 279,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.478402477441401,
      "grad_norm": 5.878256797790527,
      "learning_rate": 0.00018110068407841009,
      "logits/chosen": -0.8344768285751343,
      "logits/rejected": -0.8355271816253662,
      "logps/chosen": -7.216890335083008,
      "logps/rejected": -31.038660049438477,
      "loss": 0.6482636332511902,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3379790782928467,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.15157388150691986,
      "rewards/margins": 2.1786890029907227,
      "rewards/rejected": -2.0271151065826416,
      "step": 280,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.48011105771797746,
      "grad_norm": 3.594700336456299,
      "learning_rate": 0.00018093492829508636,
      "logits/chosen": -0.809922456741333,
      "logits/rejected": -0.8155569434165955,
      "logps/chosen": -8.301997184753418,
      "logps/rejected": -35.9427604675293,
      "loss": 0.7471737265586853,
      "memory(GiB)": 46.38,
      "nll_loss": 0.412964791059494,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.21606266498565674,
      "rewards/margins": 2.184842824935913,
      "rewards/rejected": -1.9687800407409668,
      "step": 281,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.4818196379945539,
      "grad_norm": 4.715336799621582,
      "learning_rate": 0.00018076852535823268,
      "logits/chosen": -0.8024042844772339,
      "logits/rejected": -0.8047822713851929,
      "logps/chosen": -9.50023365020752,
      "logps/rejected": -26.15119743347168,
      "loss": 0.887703001499176,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4939606785774231,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09452123939990997,
      "rewards/margins": 1.60733962059021,
      "rewards/rejected": -1.512818455696106,
      "step": 282,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.48352821827113035,
      "grad_norm": 2.6357319355010986,
      "learning_rate": 0.0001806014765984025,
      "logits/chosen": -0.7931514978408813,
      "logits/rejected": -0.7975669503211975,
      "logps/chosen": -4.370926856994629,
      "logps/rejected": -35.80842590332031,
      "loss": 0.4094904065132141,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17323070764541626,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25759178400039673,
      "rewards/margins": 2.2846627235412598,
      "rewards/rejected": -2.027071237564087,
      "step": 283,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.48523679854770674,
      "grad_norm": 4.521765232086182,
      "learning_rate": 0.00018043378335131326,
      "logits/chosen": -0.8225452899932861,
      "logits/rejected": -0.8211522698402405,
      "logps/chosen": -7.510035037994385,
      "logps/rejected": -36.8507194519043,
      "loss": 0.5709434747695923,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35384440422058105,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24606043100357056,
      "rewards/margins": 2.5631608963012695,
      "rewards/rejected": -2.3171002864837646,
      "step": 284,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4869453788242832,
      "grad_norm": 2.36458683013916,
      "learning_rate": 0.00018026544695783566,
      "logits/chosen": -0.8055735230445862,
      "logits/rejected": -0.8072599768638611,
      "logps/chosen": -8.788959503173828,
      "logps/rejected": -40.345008850097656,
      "loss": 0.5267823338508606,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34875571727752686,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.25422000885009766,
      "rewards/margins": 2.772775650024414,
      "rewards/rejected": -2.5185558795928955,
      "step": 285,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.4886539591008596,
      "grad_norm": 10.764819145202637,
      "learning_rate": 0.00018009646876398307,
      "logits/chosen": -0.8059338331222534,
      "logits/rejected": -0.8106356263160706,
      "logps/chosen": -4.930308818817139,
      "logps/rejected": -31.040836334228516,
      "loss": 0.5658331513404846,
      "memory(GiB)": 46.38,
      "nll_loss": 0.28465500473976135,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1854298710823059,
      "rewards/margins": 1.930139422416687,
      "rewards/rejected": -1.7447094917297363,
      "step": 286,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.49036253937743607,
      "grad_norm": 3.613628387451172,
      "learning_rate": 0.0001799268501209006,
      "logits/chosen": -0.7612501382827759,
      "logits/rejected": -0.7627686858177185,
      "logps/chosen": -10.456869125366211,
      "logps/rejected": -28.227718353271484,
      "loss": 0.8326478004455566,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4999573230743408,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.05842627212405205,
      "rewards/margins": 1.9171324968338013,
      "rewards/rejected": -1.8587062358856201,
      "step": 287,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.4920711196540125,
      "grad_norm": 4.662235736846924,
      "learning_rate": 0.0001797565923848544,
      "logits/chosen": -0.8021095991134644,
      "logits/rejected": -0.8046351671218872,
      "logps/chosen": -6.780048370361328,
      "logps/rejected": -31.340526580810547,
      "loss": 0.5166000723838806,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29529324173927307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24387100338935852,
      "rewards/margins": 2.386127233505249,
      "rewards/rejected": -2.142256259918213,
      "step": 288,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.4937796999305889,
      "grad_norm": 3.747556447982788,
      "learning_rate": 0.00017958569691722077,
      "logits/chosen": -0.7904728055000305,
      "logits/rejected": -0.7943806052207947,
      "logps/chosen": -4.052952766418457,
      "logps/rejected": -39.81676483154297,
      "loss": 0.444693922996521,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26461470127105713,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24267469346523285,
      "rewards/margins": 3.1456563472747803,
      "rewards/rejected": -2.9029815196990967,
      "step": 289,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.49548828020716534,
      "grad_norm": 3.8240482807159424,
      "learning_rate": 0.00017941416508447536,
      "logits/chosen": -0.8258907794952393,
      "logits/rejected": -0.8271006345748901,
      "logps/chosen": -12.335704803466797,
      "logps/rejected": -35.24673843383789,
      "loss": 0.8288266062736511,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5496087670326233,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.26559069752693176,
      "rewards/margins": 2.4179635047912598,
      "rewards/rejected": -2.1523728370666504,
      "step": 290,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.4971968604837418,
      "grad_norm": 5.618929386138916,
      "learning_rate": 0.0001792419982581821,
      "logits/chosen": -0.8527386784553528,
      "logits/rejected": -0.8554395437240601,
      "logps/chosen": -4.423219203948975,
      "logps/rejected": -37.118656158447266,
      "loss": 0.4001955986022949,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2208787500858307,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2664373517036438,
      "rewards/margins": 2.81235933303833,
      "rewards/rejected": -2.545922040939331,
      "step": 291,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.49890544076031823,
      "grad_norm": 4.9184112548828125,
      "learning_rate": 0.00017906919781498235,
      "logits/chosen": -0.8180443048477173,
      "logits/rejected": -0.8243182897567749,
      "logps/chosen": -7.195798873901367,
      "logps/rejected": -46.805763244628906,
      "loss": 0.50282222032547,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24749858677387238,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17263692617416382,
      "rewards/margins": 3.1018624305725098,
      "rewards/rejected": -2.929225444793701,
      "step": 292,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.5006140210368947,
      "grad_norm": 6.550334930419922,
      "learning_rate": 0.00017889576513658385,
      "logits/chosen": -0.8828959465026855,
      "logits/rejected": -0.8903377652168274,
      "logps/chosen": -5.666334629058838,
      "logps/rejected": -41.20561218261719,
      "loss": 0.5666110515594482,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27025288343429565,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14388155937194824,
      "rewards/margins": 2.9238851070404053,
      "rewards/rejected": -2.780003547668457,
      "step": 293,
      "train_speed(iter/s)": 0.005487
    },
    {
      "epoch": 0.5023226013134711,
      "grad_norm": 5.214411735534668,
      "learning_rate": 0.0001787217016097497,
      "logits/chosen": -0.8843887448310852,
      "logits/rejected": -0.8880648612976074,
      "logps/chosen": -8.593646049499512,
      "logps/rejected": -31.845436096191406,
      "loss": 0.7616337537765503,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4269709587097168,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.03095342591404915,
      "rewards/margins": 2.152078628540039,
      "rewards/rejected": -2.1211252212524414,
      "step": 294,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5040311815900476,
      "grad_norm": 3.419295310974121,
      "learning_rate": 0.00017854700862628717,
      "logits/chosen": -0.8697606325149536,
      "logits/rejected": -0.8711093068122864,
      "logps/chosen": -12.32712459564209,
      "logps/rejected": -35.99604415893555,
      "loss": 0.7168628573417664,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4990694522857666,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.00020820647478103638,
      "rewards/margins": 2.4814581871032715,
      "rewards/rejected": -2.481666088104248,
      "step": 295,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.505739761866624,
      "grad_norm": 5.47044038772583,
      "learning_rate": 0.00017837168758303673,
      "logits/chosen": -0.9019865989685059,
      "logits/rejected": -0.9105886220932007,
      "logps/chosen": -6.227141380310059,
      "logps/rejected": -53.5411376953125,
      "loss": 0.4358929395675659,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2536502182483673,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.18560552597045898,
      "rewards/margins": 4.006980895996094,
      "rewards/rejected": -3.8213753700256348,
      "step": 296,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5074483421432003,
      "grad_norm": 2.2059149742126465,
      "learning_rate": 0.00017819573988186077,
      "logits/chosen": -0.8752774000167847,
      "logits/rejected": -0.8806784749031067,
      "logps/chosen": -6.102680206298828,
      "logps/rejected": -40.7698860168457,
      "loss": 0.5294958353042603,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32509148120880127,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.20541568100452423,
      "rewards/margins": 2.9198334217071533,
      "rewards/rejected": -2.7144179344177246,
      "step": 297,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5091569224197768,
      "grad_norm": 1.8315078020095825,
      "learning_rate": 0.00017801916692963246,
      "logits/chosen": -0.888721764087677,
      "logits/rejected": -0.8927589654922485,
      "logps/chosen": -5.959261417388916,
      "logps/rejected": -39.12718963623047,
      "loss": 0.41302210092544556,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25391918420791626,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.36076462268829346,
      "rewards/margins": 2.8598427772521973,
      "rewards/rejected": -2.4990782737731934,
      "step": 298,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.5108655026963532,
      "grad_norm": 2.749384641647339,
      "learning_rate": 0.00017784197013822435,
      "logits/chosen": -0.8662399053573608,
      "logits/rejected": -0.8707250356674194,
      "logps/chosen": -9.147662162780762,
      "logps/rejected": -45.20575714111328,
      "loss": 0.6315680146217346,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45470985770225525,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07495459169149399,
      "rewards/margins": 3.035630226135254,
      "rewards/rejected": -2.9606757164001465,
      "step": 299,
      "train_speed(iter/s)": 0.005489
    },
    {
      "epoch": 0.5125740829729297,
      "grad_norm": 8.072540283203125,
      "learning_rate": 0.0001776641509244973,
      "logits/chosen": -0.8409525156021118,
      "logits/rejected": -0.8496344685554504,
      "logps/chosen": -6.282418251037598,
      "logps/rejected": -41.63984680175781,
      "loss": 0.871893584728241,
      "memory(GiB)": 46.38,
      "nll_loss": 0.603092610836029,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04116540029644966,
      "rewards/margins": 2.8330185413360596,
      "rewards/rejected": -2.7918529510498047,
      "step": 300,
      "train_speed(iter/s)": 0.005488
    },
    {
      "epoch": 0.5125740829729297,
      "eval_logits/chosen": -0.8405543565750122,
      "eval_logits/rejected": -0.8445731401443481,
      "eval_logps/chosen": -8.006686210632324,
      "eval_logps/rejected": -40.67301559448242,
      "eval_loss": 0.7707608342170715,
      "eval_nll_loss": 0.5364512801170349,
      "eval_rewards/accuracies": 0.8994709253311157,
      "eval_rewards/chosen": 0.06882528215646744,
      "eval_rewards/margins": 2.7935357093811035,
      "eval_rewards/rejected": -2.724710702896118,
      "eval_runtime": 442.8747,
      "eval_samples_per_second": 0.427,
      "eval_steps_per_second": 0.427,
      "step": 300
    },
    {
      "epoch": 0.5142826632495061,
      "grad_norm": 11.069442749023438,
      "learning_rate": 0.000177485710710289,
      "logits/chosen": -0.8602814674377441,
      "logits/rejected": -0.8637278079986572,
      "logps/chosen": -7.823677062988281,
      "logps/rejected": -39.174312591552734,
      "loss": 0.45352113246917725,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32754451036453247,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.15461236238479614,
      "rewards/margins": 3.001373291015625,
      "rewards/rejected": -2.8467607498168945,
      "step": 301,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 0.5159912435260825,
      "grad_norm": 4.37824821472168,
      "learning_rate": 0.0001773066509224026,
      "logits/chosen": -0.7942736744880676,
      "logits/rejected": -0.7966884970664978,
      "logps/chosen": -7.395168304443359,
      "logps/rejected": -37.037654876708984,
      "loss": 0.6269878149032593,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4148014783859253,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.27507781982421875,
      "rewards/margins": 2.9940710067749023,
      "rewards/rejected": -2.7189929485321045,
      "step": 302,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 0.517699823802659,
      "grad_norm": 1.2747763395309448,
      "learning_rate": 0.0001771269729925955,
      "logits/chosen": -0.8845440745353699,
      "logits/rejected": -0.8931821584701538,
      "logps/chosen": -4.624814510345459,
      "logps/rejected": -54.01844787597656,
      "loss": 0.3171394467353821,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2531016170978546,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3055804371833801,
      "rewards/margins": 4.609305381774902,
      "rewards/rejected": -4.303724765777588,
      "step": 303,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5194084040792354,
      "grad_norm": 19.4792423248291,
      "learning_rate": 0.00017694667835756758,
      "logits/chosen": -0.8769544363021851,
      "logits/rejected": -0.8785020112991333,
      "logps/chosen": -13.249614715576172,
      "logps/rejected": -43.8223991394043,
      "loss": 1.2327057123184204,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8601534962654114,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.30870580673217773,
      "rewards/margins": 2.7113828659057617,
      "rewards/rejected": -3.0200884342193604,
      "step": 304,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5211169843558119,
      "grad_norm": 9.179790496826172,
      "learning_rate": 0.0001767657684589499,
      "logits/chosen": -0.8855112195014954,
      "logits/rejected": -0.8912838101387024,
      "logps/chosen": -7.211820602416992,
      "logps/rejected": -50.378562927246094,
      "loss": 0.7304961085319519,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42998042702674866,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.09179924428462982,
      "rewards/margins": 4.0141072273254395,
      "rewards/rejected": -3.9223079681396484,
      "step": 305,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5228255646323883,
      "grad_norm": 1.2901352643966675,
      "learning_rate": 0.00017658424474329326,
      "logits/chosen": -0.8675682544708252,
      "logits/rejected": -0.8787212371826172,
      "logps/chosen": -6.3633317947387695,
      "logps/rejected": -68.60183715820312,
      "loss": 0.31616559624671936,
      "memory(GiB)": 46.38,
      "nll_loss": 0.265287309885025,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.21000903844833374,
      "rewards/margins": 5.580310344696045,
      "rewards/rejected": -5.370301723480225,
      "step": 306,
      "train_speed(iter/s)": 0.005444
    },
    {
      "epoch": 0.5245341449089647,
      "grad_norm": 4.6658782958984375,
      "learning_rate": 0.00017640210866205639,
      "logits/chosen": -0.906684935092926,
      "logits/rejected": -0.9144970774650574,
      "logps/chosen": -9.675024032592773,
      "logps/rejected": -51.09449768066406,
      "loss": 0.6057628989219666,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4204673767089844,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.10616739839315414,
      "rewards/margins": 3.975714921951294,
      "rewards/rejected": -4.081882476806641,
      "step": 307,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5262427251855412,
      "grad_norm": 5.465089321136475,
      "learning_rate": 0.00017621936167159458,
      "logits/chosen": -0.9267672300338745,
      "logits/rejected": -0.9362570643424988,
      "logps/chosen": -7.424238681793213,
      "logps/rejected": -65.96898651123047,
      "loss": 0.47528597712516785,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3133973479270935,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.09580647945404053,
      "rewards/margins": 5.268470287322998,
      "rewards/rejected": -5.364277362823486,
      "step": 308,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5279513054621175,
      "grad_norm": 5.624989986419678,
      "learning_rate": 0.00017603600523314787,
      "logits/chosen": -0.898685097694397,
      "logits/rejected": -0.8996263146400452,
      "logps/chosen": -12.03923225402832,
      "logps/rejected": -41.64332962036133,
      "loss": 0.7952063083648682,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4172392785549164,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.023442525416612625,
      "rewards/margins": 3.3135008811950684,
      "rewards/rejected": -3.2900586128234863,
      "step": 309,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.529659885738694,
      "grad_norm": 3.880650043487549,
      "learning_rate": 0.0001758520408128295,
      "logits/chosen": -0.8696607947349548,
      "logits/rejected": -0.8726595640182495,
      "logps/chosen": -7.42938232421875,
      "logps/rejected": -56.99799346923828,
      "loss": 0.5451213717460632,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3175729513168335,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16648581624031067,
      "rewards/margins": 4.477063179016113,
      "rewards/rejected": -4.310576915740967,
      "step": 310,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5313684660152704,
      "grad_norm": 2.883660316467285,
      "learning_rate": 0.00017566746988161402,
      "logits/chosen": -0.8406087756156921,
      "logits/rejected": -0.8471619486808777,
      "logps/chosen": -9.634123802185059,
      "logps/rejected": -38.94046401977539,
      "loss": 0.6529473662376404,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38416606187820435,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.13221141695976257,
      "rewards/margins": 3.1708903312683105,
      "rewards/rejected": -3.0386788845062256,
      "step": 311,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5330770462918468,
      "grad_norm": 11.171285629272461,
      "learning_rate": 0.00017548229391532572,
      "logits/chosen": -0.8072686791419983,
      "logits/rejected": -0.8102389574050903,
      "logps/chosen": -6.779449462890625,
      "logps/rejected": -49.09239959716797,
      "loss": 0.5393321514129639,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3658963441848755,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.2095126211643219,
      "rewards/margins": 3.693910598754883,
      "rewards/rejected": -3.4843978881835938,
      "step": 312,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5347856265684233,
      "grad_norm": 1.954391360282898,
      "learning_rate": 0.00017529651439462664,
      "logits/chosen": -0.7809416055679321,
      "logits/rejected": -0.7857435345649719,
      "logps/chosen": -6.740390777587891,
      "logps/rejected": -48.86231994628906,
      "loss": 0.36391139030456543,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2839314639568329,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31254059076309204,
      "rewards/margins": 3.944000482559204,
      "rewards/rejected": -3.6314597129821777,
      "step": 313,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5364942068449997,
      "grad_norm": 6.087454319000244,
      "learning_rate": 0.00017511013280500494,
      "logits/chosen": -0.7839592695236206,
      "logits/rejected": -0.7883790731430054,
      "logps/chosen": -5.397429943084717,
      "logps/rejected": -38.226078033447266,
      "loss": 0.47547078132629395,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2724950909614563,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16703476011753082,
      "rewards/margins": 2.7412147521972656,
      "rewards/rejected": -2.5741801261901855,
      "step": 314,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5382027871215762,
      "grad_norm": 7.710384845733643,
      "learning_rate": 0.0001749231506367628,
      "logits/chosen": -0.7206589579582214,
      "logits/rejected": -0.7282582521438599,
      "logps/chosen": -8.630600929260254,
      "logps/rejected": -40.66322708129883,
      "loss": 0.7258225679397583,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4553995132446289,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14358378946781158,
      "rewards/margins": 2.626176595687866,
      "rewards/rejected": -2.482593059539795,
      "step": 315,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5399113673981526,
      "grad_norm": 1.8784159421920776,
      "learning_rate": 0.0001747355693850047,
      "logits/chosen": -0.7431631684303284,
      "logits/rejected": -0.748499870300293,
      "logps/chosen": -11.859859466552734,
      "logps/rejected": -43.0447883605957,
      "loss": 0.6120511293411255,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4628870189189911,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1693616509437561,
      "rewards/margins": 2.6937308311462402,
      "rewards/rejected": -2.524369239807129,
      "step": 316,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.541619947674729,
      "grad_norm": 2.4822375774383545,
      "learning_rate": 0.00017454739054962527,
      "logits/chosen": -0.7296395301818848,
      "logits/rejected": -0.7350925207138062,
      "logps/chosen": -7.4225616455078125,
      "logps/rejected": -44.191368103027344,
      "loss": 0.5170390605926514,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3234650194644928,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.14995595812797546,
      "rewards/margins": 3.103259563446045,
      "rewards/rejected": -2.953303813934326,
      "step": 317,
      "train_speed(iter/s)": 0.005445
    },
    {
      "epoch": 0.5433285279513055,
      "grad_norm": 1.8158119916915894,
      "learning_rate": 0.00017435861563529746,
      "logits/chosen": -0.7801295518875122,
      "logits/rejected": -0.7813860177993774,
      "logps/chosen": -5.427134990692139,
      "logps/rejected": -36.239803314208984,
      "loss": 0.36503538489341736,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19436712563037872,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22437262535095215,
      "rewards/margins": 2.704486131668091,
      "rewards/rejected": -2.4801137447357178,
      "step": 318,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5450371082278819,
      "grad_norm": 1.2651731967926025,
      "learning_rate": 0.00017416924615146055,
      "logits/chosen": -0.7636649012565613,
      "logits/rejected": -0.7674313187599182,
      "logps/chosen": -4.830596923828125,
      "logps/rejected": -49.42289733886719,
      "loss": 0.2704404294490814,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17981436848640442,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2880362272262573,
      "rewards/margins": 3.9220433235168457,
      "rewards/rejected": -3.634007215499878,
      "step": 319,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5467456885044584,
      "grad_norm": 2.813702344894409,
      "learning_rate": 0.0001739792836123078,
      "logits/chosen": -0.763135552406311,
      "logits/rejected": -0.7641803622245789,
      "logps/chosen": -8.197504043579102,
      "logps/rejected": -46.546730041503906,
      "loss": 0.43457722663879395,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3029811978340149,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.21454845368862152,
      "rewards/margins": 3.952043056488037,
      "rewards/rejected": -3.737494707107544,
      "step": 320,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5484542687810348,
      "grad_norm": 10.072509765625,
      "learning_rate": 0.00017378872953677463,
      "logits/chosen": -0.7955141663551331,
      "logits/rejected": -0.8014116883277893,
      "logps/chosen": -6.334041595458984,
      "logps/rejected": -47.14308166503906,
      "loss": 0.4273318648338318,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26563501358032227,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.30760833621025085,
      "rewards/margins": 3.725619316101074,
      "rewards/rejected": -3.418011426925659,
      "step": 321,
      "train_speed(iter/s)": 0.005446
    },
    {
      "epoch": 0.5501628490576111,
      "grad_norm": 11.081689834594727,
      "learning_rate": 0.00017359758544852635,
      "logits/chosen": -0.8159039616584778,
      "logits/rejected": -0.8194667100906372,
      "logps/chosen": -9.277294158935547,
      "logps/rejected": -51.40448760986328,
      "loss": 0.8425000309944153,
      "memory(GiB)": 46.38,
      "nll_loss": 0.717254102230072,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.03732386231422424,
      "rewards/margins": 4.070914268493652,
      "rewards/rejected": -4.033590316772461,
      "step": 322,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5518714293341876,
      "grad_norm": 2.0417747497558594,
      "learning_rate": 0.00017340585287594604,
      "logits/chosen": -0.8340936899185181,
      "logits/rejected": -0.8355550169944763,
      "logps/chosen": -8.740560531616211,
      "logps/rejected": -48.48387908935547,
      "loss": 0.5082376599311829,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3596939742565155,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3152270019054413,
      "rewards/margins": 3.8847768306732178,
      "rewards/rejected": -3.569550037384033,
      "step": 323,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.553580009610764,
      "grad_norm": 11.896222114562988,
      "learning_rate": 0.00017321353335212215,
      "logits/chosen": -0.8562039732933044,
      "logits/rejected": -0.8574480414390564,
      "logps/chosen": -9.903059005737305,
      "logps/rejected": -48.68834686279297,
      "loss": 0.5927642583847046,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3851415812969208,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.1212349608540535,
      "rewards/margins": 3.81803560256958,
      "rewards/rejected": -3.696800708770752,
      "step": 324,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5552885898873405,
      "grad_norm": 3.4423718452453613,
      "learning_rate": 0.00017302062841483655,
      "logits/chosen": -0.8601418733596802,
      "logits/rejected": -0.8596406579017639,
      "logps/chosen": -9.926023483276367,
      "logps/rejected": -53.83884811401367,
      "loss": 0.45005056262016296,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34139543771743774,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20918819308280945,
      "rewards/margins": 4.317967891693115,
      "rewards/rejected": -4.108779430389404,
      "step": 325,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5569971701639169,
      "grad_norm": 1.2915583848953247,
      "learning_rate": 0.00017282713960655195,
      "logits/chosen": -0.9310311079025269,
      "logits/rejected": -0.9432608485221863,
      "logps/chosen": -3.527310848236084,
      "logps/rejected": -56.88557434082031,
      "loss": 0.257373183965683,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18794924020767212,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.31664133071899414,
      "rewards/margins": 4.63478422164917,
      "rewards/rejected": -4.318142890930176,
      "step": 326,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5587057504404933,
      "grad_norm": 12.496362686157227,
      "learning_rate": 0.00017263306847439968,
      "logits/chosen": -0.8998693227767944,
      "logits/rejected": -0.9072844386100769,
      "logps/chosen": -7.473057270050049,
      "logps/rejected": -62.43585968017578,
      "loss": 0.4956499934196472,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36785978078842163,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2565673589706421,
      "rewards/margins": 4.963648796081543,
      "rewards/rejected": -4.707081317901611,
      "step": 327,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5604143307170698,
      "grad_norm": 4.157985210418701,
      "learning_rate": 0.0001724384165701674,
      "logits/chosen": -0.9274346828460693,
      "logits/rejected": -0.9325964450836182,
      "logps/chosen": -7.024556636810303,
      "logps/rejected": -62.318359375,
      "loss": 0.35737261176109314,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25133123993873596,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.11644118279218674,
      "rewards/margins": 4.924075126647949,
      "rewards/rejected": -4.807634353637695,
      "step": 328,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.5621229109936462,
      "grad_norm": 4.035698413848877,
      "learning_rate": 0.00017224318545028649,
      "logits/chosen": -0.9244407415390015,
      "logits/rejected": -0.9326272010803223,
      "logps/chosen": -6.016265392303467,
      "logps/rejected": -56.14576721191406,
      "loss": 0.3880583643913269,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3103344738483429,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07886901497840881,
      "rewards/margins": 4.746619701385498,
      "rewards/rejected": -4.667750358581543,
      "step": 329,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5638314912702227,
      "grad_norm": 9.743223190307617,
      "learning_rate": 0.0001720473766758198,
      "logits/chosen": -0.926537275314331,
      "logits/rejected": -0.9304601550102234,
      "logps/chosen": -13.121172904968262,
      "logps/rejected": -58.38237380981445,
      "loss": 0.7646811008453369,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5820285677909851,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.09494726359844208,
      "rewards/margins": 4.842239856719971,
      "rewards/rejected": -4.9371867179870605,
      "step": 330,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5655400715467991,
      "grad_norm": 18.880859375,
      "learning_rate": 0.00017185099181244907,
      "logits/chosen": -0.9021615386009216,
      "logits/rejected": -0.9024704694747925,
      "logps/chosen": -9.637939453125,
      "logps/rejected": -58.98625183105469,
      "loss": 0.739081859588623,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6550078392028809,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.00911126658320427,
      "rewards/margins": 5.053913116455078,
      "rewards/rejected": -5.063024997711182,
      "step": 331,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5672486518233755,
      "grad_norm": 2.728888750076294,
      "learning_rate": 0.00017165403243046248,
      "logits/chosen": -0.9438092112541199,
      "logits/rejected": -0.9487511515617371,
      "logps/chosen": -10.97130012512207,
      "logps/rejected": -60.31357955932617,
      "loss": 0.48772644996643066,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4556671679019928,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31157898902893066,
      "rewards/margins": 5.334751129150391,
      "rewards/rejected": -5.023171901702881,
      "step": 332,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.568957232099952,
      "grad_norm": 17.81595802307129,
      "learning_rate": 0.00017145650010474194,
      "logits/chosen": -0.9099326133728027,
      "logits/rejected": -0.913872241973877,
      "logps/chosen": -7.330003261566162,
      "logps/rejected": -56.975791931152344,
      "loss": 0.44156697392463684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3481760025024414,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.13420407474040985,
      "rewards/margins": 4.564350128173828,
      "rewards/rejected": -4.430145740509033,
      "step": 333,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5706658123765284,
      "grad_norm": 4.562387466430664,
      "learning_rate": 0.00017125839641475072,
      "logits/chosen": -0.93560391664505,
      "logits/rejected": -0.9402634501457214,
      "logps/chosen": -8.35469913482666,
      "logps/rejected": -48.29425048828125,
      "loss": 0.8423429727554321,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5765202045440674,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20604895055294037,
      "rewards/margins": 4.131405353546143,
      "rewards/rejected": -3.925356149673462,
      "step": 334,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5723743926531049,
      "grad_norm": 6.9514899253845215,
      "learning_rate": 0.00017105972294452056,
      "logits/chosen": -0.9457940459251404,
      "logits/rejected": -0.9509940147399902,
      "logps/chosen": -8.684478759765625,
      "logps/rejected": -42.20777893066406,
      "loss": 0.6009758114814758,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2808820307254791,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.007849723100662231,
      "rewards/margins": 3.273137092590332,
      "rewards/rejected": -3.265286922454834,
      "step": 335,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5740829729296812,
      "grad_norm": 41.98228454589844,
      "learning_rate": 0.0001708604812826393,
      "logits/chosen": -0.9295578598976135,
      "logits/rejected": -0.935192346572876,
      "logps/chosen": -9.62477970123291,
      "logps/rejected": -54.46697235107422,
      "loss": 0.598048746585846,
      "memory(GiB)": 46.38,
      "nll_loss": 0.43297702074050903,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07474887371063232,
      "rewards/margins": 4.1404008865356445,
      "rewards/rejected": -4.065651893615723,
      "step": 336,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5757915532062576,
      "grad_norm": 2.598214626312256,
      "learning_rate": 0.00017066067302223798,
      "logits/chosen": -0.8595199584960938,
      "logits/rejected": -0.8645521402359009,
      "logps/chosen": -5.353518962860107,
      "logps/rejected": -45.4082145690918,
      "loss": 0.3949076235294342,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3393931984901428,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17312097549438477,
      "rewards/margins": 3.7592127323150635,
      "rewards/rejected": -3.5860915184020996,
      "step": 337,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5775001334828341,
      "grad_norm": 6.433867454528809,
      "learning_rate": 0.00017046029976097805,
      "logits/chosen": -0.8666653633117676,
      "logits/rejected": -0.8727781176567078,
      "logps/chosen": -7.716047286987305,
      "logps/rejected": -44.40462112426758,
      "loss": 0.7068074345588684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4196939468383789,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.018908580765128136,
      "rewards/margins": 3.310499429702759,
      "rewards/rejected": -3.291590929031372,
      "step": 338,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5792087137594105,
      "grad_norm": 4.851775169372559,
      "learning_rate": 0.00017025936310103882,
      "logits/chosen": -0.8393141627311707,
      "logits/rejected": -0.8419257402420044,
      "logps/chosen": -10.38674259185791,
      "logps/rejected": -41.55541229248047,
      "loss": 0.5500025749206543,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35587984323501587,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21891042590141296,
      "rewards/margins": 3.442535400390625,
      "rewards/rejected": -3.2236249446868896,
      "step": 339,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.580917294035987,
      "grad_norm": 5.706538200378418,
      "learning_rate": 0.0001700578646491045,
      "logits/chosen": -0.849159300327301,
      "logits/rejected": -0.8539773225784302,
      "logps/chosen": -5.776857852935791,
      "logps/rejected": -41.82091522216797,
      "loss": 0.4652997851371765,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1927475780248642,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1367129385471344,
      "rewards/margins": 2.951298952102661,
      "rewards/rejected": -2.8145859241485596,
      "step": 340,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.5826258743125634,
      "grad_norm": 15.33443832397461,
      "learning_rate": 0.00016985580601635124,
      "logits/chosen": -0.8022753000259399,
      "logits/rejected": -0.809974193572998,
      "logps/chosen": -10.902517318725586,
      "logps/rejected": -38.2662467956543,
      "loss": 0.7963184118270874,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5153663158416748,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.12418508529663086,
      "rewards/margins": 2.705582618713379,
      "rewards/rejected": -2.58139705657959,
      "step": 341,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5843344545891398,
      "grad_norm": 2.2032649517059326,
      "learning_rate": 0.00016965318881843457,
      "logits/chosen": -0.7673312425613403,
      "logits/rejected": -0.7743449211120605,
      "logps/chosen": -8.663345336914062,
      "logps/rejected": -45.02417755126953,
      "loss": 0.5525592565536499,
      "memory(GiB)": 46.38,
      "nll_loss": 0.42993825674057007,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.231624573469162,
      "rewards/margins": 3.4039034843444824,
      "rewards/rejected": -3.172279119491577,
      "step": 342,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.5860430348657163,
      "grad_norm": 2.8073010444641113,
      "learning_rate": 0.0001694500146754762,
      "logits/chosen": -0.8122238516807556,
      "logits/rejected": -0.8119744658470154,
      "logps/chosen": -8.633512496948242,
      "logps/rejected": -35.36651611328125,
      "loss": 0.5415773391723633,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31399714946746826,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.15843312442302704,
      "rewards/margins": 2.734858989715576,
      "rewards/rejected": -2.576425790786743,
      "step": 343,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5877516151422927,
      "grad_norm": 2.3773810863494873,
      "learning_rate": 0.0001692462852120511,
      "logits/chosen": -0.7741044759750366,
      "logits/rejected": -0.7800744771957397,
      "logps/chosen": -9.291975021362305,
      "logps/rejected": -40.77639389038086,
      "loss": 0.5208418965339661,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3624192774295807,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2715660035610199,
      "rewards/margins": 2.76344633102417,
      "rewards/rejected": -2.491880416870117,
      "step": 344,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5894601954188692,
      "grad_norm": 5.773314952850342,
      "learning_rate": 0.0001690420020571747,
      "logits/chosen": -0.7945303916931152,
      "logits/rejected": -0.8053185343742371,
      "logps/chosen": -7.3475542068481445,
      "logps/rejected": -50.37881851196289,
      "loss": 0.42357194423675537,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32000547647476196,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3487846553325653,
      "rewards/margins": 3.4395201206207275,
      "rewards/rejected": -3.090735673904419,
      "step": 345,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5911687756954456,
      "grad_norm": 6.197271347045898,
      "learning_rate": 0.0001688371668442896,
      "logits/chosen": -0.805290937423706,
      "logits/rejected": -0.8079847097396851,
      "logps/chosen": -7.421670436859131,
      "logps/rejected": -37.118797302246094,
      "loss": 0.5294437408447266,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3508721590042114,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17643693089485168,
      "rewards/margins": 2.624368667602539,
      "rewards/rejected": -2.4479315280914307,
      "step": 346,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.592877355972022,
      "grad_norm": 3.835904836654663,
      "learning_rate": 0.00016863178121125276,
      "logits/chosen": -0.8487049341201782,
      "logits/rejected": -0.8516339063644409,
      "logps/chosen": -8.74366283416748,
      "logps/rejected": -34.4682502746582,
      "loss": 0.5114731788635254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3230685293674469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0005252091214060783,
      "rewards/margins": 2.4223427772521973,
      "rewards/rejected": -2.4218177795410156,
      "step": 347,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.5945859362485985,
      "grad_norm": 14.421294212341309,
      "learning_rate": 0.00016842584680032223,
      "logits/chosen": -0.8961611986160278,
      "logits/rejected": -0.9048024415969849,
      "logps/chosen": -5.175715923309326,
      "logps/rejected": -48.69798278808594,
      "loss": 0.7653312683105469,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5099776387214661,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.08455780148506165,
      "rewards/margins": 3.585397720336914,
      "rewards/rejected": -3.5008397102355957,
      "step": 348,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5962945165251748,
      "grad_norm": 3.0503435134887695,
      "learning_rate": 0.000168219365258144,
      "logits/chosen": -0.9129156470298767,
      "logits/rejected": -0.9179389476776123,
      "logps/chosen": -10.671258926391602,
      "logps/rejected": -43.36078643798828,
      "loss": 0.37005069851875305,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25679123401641846,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24289637804031372,
      "rewards/margins": 3.3170793056488037,
      "rewards/rejected": -3.0741827487945557,
      "step": 349,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5980030968017513,
      "grad_norm": 21.91331672668457,
      "learning_rate": 0.00016801233823573908,
      "logits/chosen": -0.8929505944252014,
      "logits/rejected": -0.8917251229286194,
      "logps/chosen": -14.047558784484863,
      "logps/rejected": -31.77324676513672,
      "loss": 0.976524829864502,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6302264928817749,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11739279329776764,
      "rewards/margins": 2.250826120376587,
      "rewards/rejected": -2.1334333419799805,
      "step": 350,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.5997116770783277,
      "grad_norm": 1.7703797817230225,
      "learning_rate": 0.00016780476738848993,
      "logits/chosen": -0.9439638257026672,
      "logits/rejected": -0.9439706206321716,
      "logps/chosen": -8.544902801513672,
      "logps/rejected": -40.44263458251953,
      "loss": 0.4238983690738678,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29430148005485535,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26157146692276,
      "rewards/margins": 3.2353665828704834,
      "rewards/rejected": -2.973795175552368,
      "step": 351,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.6014202573549041,
      "grad_norm": 3.624192953109741,
      "learning_rate": 0.00016759665437612758,
      "logits/chosen": -0.8981308937072754,
      "logits/rejected": -0.9056174159049988,
      "logps/chosen": -5.872384071350098,
      "logps/rejected": -49.91468811035156,
      "loss": 0.30830851197242737,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2274772971868515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.18507812917232513,
      "rewards/margins": 3.749619245529175,
      "rewards/rejected": -3.5645413398742676,
      "step": 352,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.6031288376314806,
      "grad_norm": 11.304423332214355,
      "learning_rate": 0.00016738800086271813,
      "logits/chosen": -0.9281446933746338,
      "logits/rejected": -0.9346886873245239,
      "logps/chosen": -6.601749420166016,
      "logps/rejected": -49.165306091308594,
      "loss": 0.5459189414978027,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3969055414199829,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.10995268821716309,
      "rewards/margins": 3.520914316177368,
      "rewards/rejected": -3.410961627960205,
      "step": 353,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.604837417908057,
      "grad_norm": 5.619907379150391,
      "learning_rate": 0.00016717880851664947,
      "logits/chosen": -0.9121412634849548,
      "logits/rejected": -0.9164290428161621,
      "logps/chosen": -9.720386505126953,
      "logps/rejected": -48.402774810791016,
      "loss": 0.9874699115753174,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6796480417251587,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.04260212555527687,
      "rewards/margins": 3.240419387817383,
      "rewards/rejected": -3.2830214500427246,
      "step": 354,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.6065459981846335,
      "grad_norm": 5.572713375091553,
      "learning_rate": 0.00016696907901061803,
      "logits/chosen": -0.9459649324417114,
      "logits/rejected": -0.9470463395118713,
      "logps/chosen": -9.30247974395752,
      "logps/rejected": -41.718082427978516,
      "loss": 0.48700881004333496,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3246360719203949,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19217610359191895,
      "rewards/margins": 3.3326284885406494,
      "rewards/rejected": -3.1404523849487305,
      "step": 355,
      "train_speed(iter/s)": 0.005447
    },
    {
      "epoch": 0.6082545784612099,
      "grad_norm": 4.559499740600586,
      "learning_rate": 0.00016675881402161536,
      "logits/chosen": -0.9564660787582397,
      "logits/rejected": -0.956013023853302,
      "logps/chosen": -7.110748291015625,
      "logps/rejected": -35.393348693847656,
      "loss": 0.46305030584335327,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2555083632469177,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.11240166425704956,
      "rewards/margins": 2.7594783306121826,
      "rewards/rejected": -2.647076368331909,
      "step": 356,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6099631587377863,
      "grad_norm": 2.224273681640625,
      "learning_rate": 0.00016654801523091473,
      "logits/chosen": -0.9595233201980591,
      "logits/rejected": -0.9649782180786133,
      "logps/chosen": -3.5286672115325928,
      "logps/rejected": -45.002864837646484,
      "loss": 0.29492613673210144,
      "memory(GiB)": 46.38,
      "nll_loss": 0.15800835192203522,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21021459996700287,
      "rewards/margins": 3.552177906036377,
      "rewards/rejected": -3.341963291168213,
      "step": 357,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6116717390143628,
      "grad_norm": 4.733156681060791,
      "learning_rate": 0.00016633668432405765,
      "logits/chosen": -0.9549999833106995,
      "logits/rejected": -0.9575210809707642,
      "logps/chosen": -7.7549333572387695,
      "logps/rejected": -41.00201416015625,
      "loss": 0.4725136160850525,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24933533370494843,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25527143478393555,
      "rewards/margins": 2.993086576461792,
      "rewards/rejected": -2.7378156185150146,
      "step": 358,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6133803192909392,
      "grad_norm": 2.0917019844055176,
      "learning_rate": 0.0001661248229908404,
      "logits/chosen": -0.9107139110565186,
      "logits/rejected": -0.9176012873649597,
      "logps/chosen": -3.37815260887146,
      "logps/rejected": -42.227508544921875,
      "loss": 0.3074043393135071,
      "memory(GiB)": 46.38,
      "nll_loss": 0.14805781841278076,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25100257992744446,
      "rewards/margins": 3.3243298530578613,
      "rewards/rejected": -3.0733275413513184,
      "step": 359,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6150888995675157,
      "grad_norm": 1.7864688634872437,
      "learning_rate": 0.00016591243292530058,
      "logits/chosen": -0.9290763139724731,
      "logits/rejected": -0.9319502115249634,
      "logps/chosen": -6.204026222229004,
      "logps/rejected": -41.98524856567383,
      "loss": 0.3325427174568176,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25362634658813477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.27647268772125244,
      "rewards/margins": 3.309528112411499,
      "rewards/rejected": -3.033055305480957,
      "step": 360,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6167974798440921,
      "grad_norm": 5.563055992126465,
      "learning_rate": 0.00016569951582570344,
      "logits/chosen": -0.9435633420944214,
      "logits/rejected": -0.9498572945594788,
      "logps/chosen": -10.231161117553711,
      "logps/rejected": -46.89916229248047,
      "loss": 0.42046236991882324,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3107416033744812,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.290690153837204,
      "rewards/margins": 3.564943790435791,
      "rewards/rejected": -3.274254322052002,
      "step": 361,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6185060601206684,
      "grad_norm": 19.7362060546875,
      "learning_rate": 0.00016548607339452853,
      "logits/chosen": -0.9261251091957092,
      "logits/rejected": -0.9280377626419067,
      "logps/chosen": -7.509496688842773,
      "logps/rejected": -46.83708190917969,
      "loss": 0.6300509572029114,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4706030786037445,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.224940225481987,
      "rewards/margins": 3.8618416786193848,
      "rewards/rejected": -3.636901617050171,
      "step": 362,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6202146403972449,
      "grad_norm": 4.567323207855225,
      "learning_rate": 0.00016527210733845572,
      "logits/chosen": -0.948748767375946,
      "logits/rejected": -0.9545145630836487,
      "logps/chosen": -5.9083099365234375,
      "logps/rejected": -51.780113220214844,
      "loss": 0.3714110255241394,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30310437083244324,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.294877290725708,
      "rewards/margins": 4.077692985534668,
      "rewards/rejected": -3.782815933227539,
      "step": 363,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6219232206738213,
      "grad_norm": 9.299772262573242,
      "learning_rate": 0.000165057619368352,
      "logits/chosen": -0.9592850208282471,
      "logits/rejected": -0.9655886888504028,
      "logps/chosen": -7.505143642425537,
      "logps/rejected": -49.48500442504883,
      "loss": 0.6307746171951294,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4708867073059082,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.0261047575622797,
      "rewards/margins": 3.699296474456787,
      "rewards/rejected": -3.673191547393799,
      "step": 364,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6236318009503978,
      "grad_norm": 2.9054408073425293,
      "learning_rate": 0.00016484261119925742,
      "logits/chosen": -1.0252788066864014,
      "logits/rejected": -1.0359681844711304,
      "logps/chosen": -6.5622758865356445,
      "logps/rejected": -51.86637496948242,
      "loss": 0.40897446870803833,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3205597996711731,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3777948021888733,
      "rewards/margins": 4.308453559875488,
      "rewards/rejected": -3.9306585788726807,
      "step": 365,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6253403812269742,
      "grad_norm": 16.93988800048828,
      "learning_rate": 0.00016462708455037157,
      "logits/chosen": -1.0015932321548462,
      "logits/rejected": -1.0080609321594238,
      "logps/chosen": -10.308164596557617,
      "logps/rejected": -46.67601776123047,
      "loss": 0.6905434131622314,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4495546817779541,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.020374327898025513,
      "rewards/margins": 3.7047128677368164,
      "rewards/rejected": -3.7250871658325195,
      "step": 366,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6270489615035506,
      "grad_norm": 1.4071120023727417,
      "learning_rate": 0.0001644110411450398,
      "logits/chosen": -0.9903697967529297,
      "logits/rejected": -0.9909346103668213,
      "logps/chosen": -7.702688217163086,
      "logps/rejected": -49.382568359375,
      "loss": 0.37644410133361816,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29211539030075073,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2812228202819824,
      "rewards/margins": 4.183906555175781,
      "rewards/rejected": -3.9026834964752197,
      "step": 367,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6287575417801271,
      "grad_norm": 2.600391387939453,
      "learning_rate": 0.00016419448271073938,
      "logits/chosen": -1.0102006196975708,
      "logits/rejected": -1.0203218460083008,
      "logps/chosen": -6.021239280700684,
      "logps/rejected": -59.3431282043457,
      "loss": 0.3945213854312897,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3236709535121918,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27218496799468994,
      "rewards/margins": 4.844854831695557,
      "rewards/rejected": -4.57266902923584,
      "step": 368,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6304661220567035,
      "grad_norm": 4.555199146270752,
      "learning_rate": 0.00016397741097906585,
      "logits/chosen": -1.0172926187515259,
      "logits/rejected": -1.025512456893921,
      "logps/chosen": -8.92133617401123,
      "logps/rejected": -58.655067443847656,
      "loss": 0.44248220324516296,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3216467499732971,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.017364632338285446,
      "rewards/margins": 4.551565170288086,
      "rewards/rejected": -4.534200191497803,
      "step": 369,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.63217470233328,
      "grad_norm": 4.0335259437561035,
      "learning_rate": 0.00016375982768571893,
      "logits/chosen": -1.0108150243759155,
      "logits/rejected": -1.0135056972503662,
      "logps/chosen": -12.687119483947754,
      "logps/rejected": -51.62519454956055,
      "loss": 0.5401836037635803,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45632433891296387,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.311867356300354,
      "rewards/margins": 4.409842014312744,
      "rewards/rejected": -4.0979743003845215,
      "step": 370,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6338832826098564,
      "grad_norm": 7.506470680236816,
      "learning_rate": 0.00016354173457048892,
      "logits/chosen": -1.0019760131835938,
      "logits/rejected": -1.004393458366394,
      "logps/chosen": -12.871381759643555,
      "logps/rejected": -45.352264404296875,
      "loss": 0.7025041580200195,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4446451663970947,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.026040881872177124,
      "rewards/margins": 3.574916124343872,
      "rewards/rejected": -3.548874855041504,
      "step": 371,
      "train_speed(iter/s)": 0.005448
    },
    {
      "epoch": 0.6355918628864328,
      "grad_norm": 1.648148536682129,
      "learning_rate": 0.00016332313337724243,
      "logits/chosen": -0.9710088968276978,
      "logits/rejected": -0.9783718585968018,
      "logps/chosen": -4.712439060211182,
      "logps/rejected": -58.88814926147461,
      "loss": 0.2675246000289917,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20625513792037964,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.10673172771930695,
      "rewards/margins": 4.859548091888428,
      "rewards/rejected": -4.752816200256348,
      "step": 372,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6373004431630093,
      "grad_norm": 6.159119129180908,
      "learning_rate": 0.0001631040258539089,
      "logits/chosen": -0.9410051703453064,
      "logits/rejected": -0.9530994892120361,
      "logps/chosen": -5.43995475769043,
      "logps/rejected": -49.19166946411133,
      "loss": 0.4198155403137207,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30013421177864075,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3105510175228119,
      "rewards/margins": 3.69779109954834,
      "rewards/rejected": -3.387240409851074,
      "step": 373,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6390090234395857,
      "grad_norm": 7.260493755340576,
      "learning_rate": 0.00016288441375246615,
      "logits/chosen": -0.9339801669120789,
      "logits/rejected": -0.9435940384864807,
      "logps/chosen": -8.451923370361328,
      "logps/rejected": -51.399906158447266,
      "loss": 0.6680124998092651,
      "memory(GiB)": 46.38,
      "nll_loss": 0.48599013686180115,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.038936566561460495,
      "rewards/margins": 3.561919927597046,
      "rewards/rejected": -3.5229835510253906,
      "step": 374,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6407176037161622,
      "grad_norm": 4.131718158721924,
      "learning_rate": 0.00016266429882892677,
      "logits/chosen": -0.9294899702072144,
      "logits/rejected": -0.9351480007171631,
      "logps/chosen": -6.996545791625977,
      "logps/rejected": -50.252662658691406,
      "loss": 0.37451890110969543,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2939547896385193,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1411602944135666,
      "rewards/margins": 3.8655099868774414,
      "rewards/rejected": -3.7243497371673584,
      "step": 375,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6424261839927385,
      "grad_norm": 16.996421813964844,
      "learning_rate": 0.00016244368284332376,
      "logits/chosen": -0.9045725464820862,
      "logits/rejected": -0.9046626687049866,
      "logps/chosen": -13.596282958984375,
      "logps/rejected": -47.78284454345703,
      "loss": 1.2898331880569458,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8870378732681274,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.1088126003742218,
      "rewards/margins": 3.4060020446777344,
      "rewards/rejected": -3.5148143768310547,
      "step": 376,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6441347642693149,
      "grad_norm": 9.713418006896973,
      "learning_rate": 0.0001622225675596967,
      "logits/chosen": -0.8765025734901428,
      "logits/rejected": -0.8799088001251221,
      "logps/chosen": -5.245368480682373,
      "logps/rejected": -44.48738479614258,
      "loss": 0.6239938735961914,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3923470377922058,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.02824319526553154,
      "rewards/margins": 3.4200828075408936,
      "rewards/rejected": -3.3918397426605225,
      "step": 377,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6458433445458914,
      "grad_norm": 16.6881103515625,
      "learning_rate": 0.00016200095474607753,
      "logits/chosen": -0.9629702568054199,
      "logits/rejected": -0.9680634140968323,
      "logps/chosen": -6.022677898406982,
      "logps/rejected": -43.582828521728516,
      "loss": 0.7116013169288635,
      "memory(GiB)": 46.38,
      "nll_loss": 0.45455941557884216,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.06634767353534698,
      "rewards/margins": 3.388127565383911,
      "rewards/rejected": -3.32177996635437,
      "step": 378,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6475519248224678,
      "grad_norm": 13.382901191711426,
      "learning_rate": 0.00016177884617447634,
      "logits/chosen": -0.9445232152938843,
      "logits/rejected": -0.9476342797279358,
      "logps/chosen": -7.114291191101074,
      "logps/rejected": -50.578712463378906,
      "loss": 0.5752607583999634,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39851492643356323,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05524886026978493,
      "rewards/margins": 3.86979603767395,
      "rewards/rejected": -3.814547061920166,
      "step": 379,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6492605050990443,
      "grad_norm": 7.912006855010986,
      "learning_rate": 0.00016155624362086739,
      "logits/chosen": -0.9427142143249512,
      "logits/rejected": -0.9560413360595703,
      "logps/chosen": -9.481163024902344,
      "logps/rejected": -53.17366409301758,
      "loss": 0.647068977355957,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4320647716522217,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.019453082233667374,
      "rewards/margins": 3.9523825645446777,
      "rewards/rejected": -3.932929277420044,
      "step": 380,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6509690853756207,
      "grad_norm": 1.963860034942627,
      "learning_rate": 0.0001613331488651747,
      "logits/chosen": -0.9658269286155701,
      "logits/rejected": -0.963149905204773,
      "logps/chosen": -9.277081489562988,
      "logps/rejected": -43.1500129699707,
      "loss": 0.39668354392051697,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3253212571144104,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2724382281303406,
      "rewards/margins": 3.834010362625122,
      "rewards/rejected": -3.561572313308716,
      "step": 381,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6526776656521971,
      "grad_norm": 3.8738293647766113,
      "learning_rate": 0.00016110956369125813,
      "logits/chosen": -0.9740622043609619,
      "logits/rejected": -0.9775519371032715,
      "logps/chosen": -8.24023723602295,
      "logps/rejected": -41.697147369384766,
      "loss": 0.4844883978366852,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2701917886734009,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.011540266685187817,
      "rewards/margins": 3.1645491123199463,
      "rewards/rejected": -3.1530089378356934,
      "step": 382,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6543862459287736,
      "grad_norm": 1.4099937677383423,
      "learning_rate": 0.00016088548988689867,
      "logits/chosen": -0.9694156050682068,
      "logits/rejected": -0.9768838882446289,
      "logps/chosen": -4.559173107147217,
      "logps/rejected": -51.46935272216797,
      "loss": 0.271286278963089,
      "memory(GiB)": 46.38,
      "nll_loss": 0.169478639960289,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.23250631988048553,
      "rewards/margins": 3.9228291511535645,
      "rewards/rejected": -3.6903223991394043,
      "step": 383,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.65609482620535,
      "grad_norm": 1.7601426839828491,
      "learning_rate": 0.00016066092924378448,
      "logits/chosen": -0.9804710745811462,
      "logits/rejected": -0.9870820641517639,
      "logps/chosen": -4.856400966644287,
      "logps/rejected": -52.06618118286133,
      "loss": 0.32891860604286194,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2294139713048935,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22404271364212036,
      "rewards/margins": 4.277286529541016,
      "rewards/rejected": -4.053244113922119,
      "step": 384,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6578034064819265,
      "grad_norm": 2.4365975856781006,
      "learning_rate": 0.00016043588355749654,
      "logits/chosen": -1.0200928449630737,
      "logits/rejected": -1.028275728225708,
      "logps/chosen": -5.5803656578063965,
      "logps/rejected": -51.13899230957031,
      "loss": 0.3513195514678955,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2133343517780304,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20496222376823425,
      "rewards/margins": 4.234604835510254,
      "rewards/rejected": -4.029642581939697,
      "step": 385,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6595119867585029,
      "grad_norm": 2.165262460708618,
      "learning_rate": 0.0001602103546274941,
      "logits/chosen": -0.9947616457939148,
      "logits/rejected": -1.0020798444747925,
      "logps/chosen": -4.793583869934082,
      "logps/rejected": -45.36370086669922,
      "loss": 0.35281091928482056,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26595374941825867,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38576024770736694,
      "rewards/margins": 3.7791500091552734,
      "rewards/rejected": -3.3933897018432617,
      "step": 386,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6612205670350793,
      "grad_norm": 2.079153060913086,
      "learning_rate": 0.0001599843442571005,
      "logits/chosen": -1.0367109775543213,
      "logits/rejected": -1.0368907451629639,
      "logps/chosen": -7.8156890869140625,
      "logps/rejected": -47.61594009399414,
      "loss": 0.4629322588443756,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3201110363006592,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32289010286331177,
      "rewards/margins": 4.081589221954346,
      "rewards/rejected": -3.7586989402770996,
      "step": 387,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6629291473116558,
      "grad_norm": 2.563002586364746,
      "learning_rate": 0.00015975785425348868,
      "logits/chosen": -1.019775152206421,
      "logits/rejected": -1.0254132747650146,
      "logps/chosen": -9.448254585266113,
      "logps/rejected": -49.63591003417969,
      "loss": 0.46262699365615845,
      "memory(GiB)": 46.38,
      "nll_loss": 0.371066153049469,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23315003514289856,
      "rewards/margins": 4.063310623168945,
      "rewards/rejected": -3.83016037940979,
      "step": 388,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.6646377275882321,
      "grad_norm": 6.976690769195557,
      "learning_rate": 0.0001595308864276666,
      "logits/chosen": -0.9950034618377686,
      "logits/rejected": -0.9960756301879883,
      "logps/chosen": -7.069814682006836,
      "logps/rejected": -42.814823150634766,
      "loss": 0.6936900019645691,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4904642701148987,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.051315054297447205,
      "rewards/margins": 3.4438416957855225,
      "rewards/rejected": -3.392526626586914,
      "step": 389,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6663463078648086,
      "grad_norm": 7.357374668121338,
      "learning_rate": 0.00015930344259446302,
      "logits/chosen": -0.9969927072525024,
      "logits/rejected": -1.006578803062439,
      "logps/chosen": -7.5834503173828125,
      "logps/rejected": -58.15363693237305,
      "loss": 0.8980169296264648,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6278271079063416,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.022799715399742126,
      "rewards/margins": 4.518400192260742,
      "rewards/rejected": -4.541200160980225,
      "step": 390,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.668054888141385,
      "grad_norm": 5.783321857452393,
      "learning_rate": 0.0001590755245725127,
      "logits/chosen": -1.025513768196106,
      "logits/rejected": -1.0348174571990967,
      "logps/chosen": -6.0355730056762695,
      "logps/rejected": -56.777584075927734,
      "loss": 0.5514633059501648,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39650726318359375,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.07000410556793213,
      "rewards/margins": 4.3388519287109375,
      "rewards/rejected": -4.268847942352295,
      "step": 391,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6697634684179614,
      "grad_norm": 18.076496124267578,
      "learning_rate": 0.00015884713418424208,
      "logits/chosen": -1.0282799005508423,
      "logits/rejected": -1.0340526103973389,
      "logps/chosen": -6.577209949493408,
      "logps/rejected": -57.57218933105469,
      "loss": 0.5178098082542419,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39848998188972473,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2828572690486908,
      "rewards/margins": 4.846663951873779,
      "rewards/rejected": -4.563807010650635,
      "step": 392,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6714720486945379,
      "grad_norm": 1.7601821422576904,
      "learning_rate": 0.00015861827325585468,
      "logits/chosen": -1.0324304103851318,
      "logits/rejected": -1.0410029888153076,
      "logps/chosen": -8.706206321716309,
      "logps/rejected": -64.2054443359375,
      "loss": 0.43514567613601685,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34488645195961,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20344428718090057,
      "rewards/margins": 5.270512580871582,
      "rewards/rejected": -5.067069053649902,
      "step": 393,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6731806289711143,
      "grad_norm": 1.440259575843811,
      "learning_rate": 0.00015838894361731637,
      "logits/chosen": -1.0355671644210815,
      "logits/rejected": -1.0532475709915161,
      "logps/chosen": -8.040314674377441,
      "logps/rejected": -85.86071014404297,
      "loss": 0.32451531291007996,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3027491569519043,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3317539393901825,
      "rewards/margins": 7.262915134429932,
      "rewards/rejected": -6.931160926818848,
      "step": 394,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6748892092476908,
      "grad_norm": 18.8165340423584,
      "learning_rate": 0.00015815914710234075,
      "logits/chosen": -1.0662992000579834,
      "logits/rejected": -1.0835834741592407,
      "logps/chosen": -11.277642250061035,
      "logps/rejected": -72.82504272460938,
      "loss": 1.035059928894043,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7357515096664429,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.13307660818099976,
      "rewards/margins": 5.558894157409668,
      "rewards/rejected": -5.691970348358154,
      "step": 395,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6765977895242672,
      "grad_norm": 42.015506744384766,
      "learning_rate": 0.0001579288855483747,
      "logits/chosen": -1.0829132795333862,
      "logits/rejected": -1.0891735553741455,
      "logps/chosen": -8.876832008361816,
      "logps/rejected": -62.791542053222656,
      "loss": 0.5757201910018921,
      "memory(GiB)": 46.38,
      "nll_loss": 0.421648234128952,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07643651217222214,
      "rewards/margins": 5.356234073638916,
      "rewards/rejected": -5.279797554016113,
      "step": 396,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6783063698008436,
      "grad_norm": 7.632444381713867,
      "learning_rate": 0.00015769816079658347,
      "logits/chosen": -1.0507287979125977,
      "logits/rejected": -1.0656074285507202,
      "logps/chosen": -7.989803791046143,
      "logps/rejected": -83.72140502929688,
      "loss": 0.43804171681404114,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3684487044811249,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30719032883644104,
      "rewards/margins": 7.238983154296875,
      "rewards/rejected": -6.931793212890625,
      "step": 397,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6800149500774201,
      "grad_norm": 7.3168182373046875,
      "learning_rate": 0.000157466974691836,
      "logits/chosen": -1.0599204301834106,
      "logits/rejected": -1.0610237121582031,
      "logps/chosen": -9.176435470581055,
      "logps/rejected": -59.7401123046875,
      "loss": 0.6606624722480774,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3968597650527954,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.018879584968090057,
      "rewards/margins": 4.99215841293335,
      "rewards/rejected": -4.973278522491455,
      "step": 398,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6817235303539965,
      "grad_norm": 2.761538028717041,
      "learning_rate": 0.00015723532908269022,
      "logits/chosen": -1.0245918035507202,
      "logits/rejected": -1.041639804840088,
      "logps/chosen": -4.204854965209961,
      "logps/rejected": -68.9018783569336,
      "loss": 0.31733036041259766,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2095213085412979,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.31564220786094666,
      "rewards/margins": 5.7434844970703125,
      "rewards/rejected": -5.427842140197754,
      "step": 399,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6834321106305729,
      "grad_norm": 14.492884635925293,
      "learning_rate": 0.00015700322582137827,
      "logits/chosen": -1.020153522491455,
      "logits/rejected": -1.0266611576080322,
      "logps/chosen": -10.99346923828125,
      "logps/rejected": -52.823726654052734,
      "loss": 0.6980295181274414,
      "memory(GiB)": 46.38,
      "nll_loss": 0.436104953289032,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16545000672340393,
      "rewards/margins": 4.334076881408691,
      "rewards/rejected": -4.16862678527832,
      "step": 400,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6851406909071494,
      "grad_norm": 2.4393866062164307,
      "learning_rate": 0.00015677066676379165,
      "logits/chosen": -1.0425410270690918,
      "logits/rejected": -1.0531504154205322,
      "logps/chosen": -5.451269149780273,
      "logps/rejected": -56.3714714050293,
      "loss": 0.48248323798179626,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39080309867858887,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3740793466567993,
      "rewards/margins": 4.812845230102539,
      "rewards/rejected": -4.438766002655029,
      "step": 401,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6868492711837257,
      "grad_norm": 1.559989094734192,
      "learning_rate": 0.00015653765376946637,
      "logits/chosen": -0.9974856972694397,
      "logits/rejected": -1.0161300897598267,
      "logps/chosen": -4.6968889236450195,
      "logps/rejected": -73.27117919921875,
      "loss": 0.25755736231803894,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17882776260375977,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32737916707992554,
      "rewards/margins": 5.956256866455078,
      "rewards/rejected": -5.62887716293335,
      "step": 402,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6885578514603022,
      "grad_norm": 10.118622779846191,
      "learning_rate": 0.0001563041887015681,
      "logits/chosen": -1.011096715927124,
      "logits/rejected": -1.0212759971618652,
      "logps/chosen": -7.617461204528809,
      "logps/rejected": -52.1561279296875,
      "loss": 0.5372264385223389,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3799927532672882,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.24851557612419128,
      "rewards/margins": 4.270379066467285,
      "rewards/rejected": -4.021862983703613,
      "step": 403,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6902664317368786,
      "grad_norm": 3.3534741401672363,
      "learning_rate": 0.00015607027342687735,
      "logits/chosen": -0.9900127053260803,
      "logits/rejected": -0.9951677918434143,
      "logps/chosen": -8.410379409790039,
      "logps/rejected": -56.12778091430664,
      "loss": 0.7923538684844971,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5614383816719055,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.047435253858566284,
      "rewards/margins": 4.5187482833862305,
      "rewards/rejected": -4.566183567047119,
      "step": 404,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.691975012013455,
      "grad_norm": 4.066547393798828,
      "learning_rate": 0.0001558359098157744,
      "logits/chosen": -1.0014097690582275,
      "logits/rejected": -1.0051602125167847,
      "logps/chosen": -6.213657379150391,
      "logps/rejected": -57.68375778198242,
      "loss": 0.3043290674686432,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23996210098266602,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17297674715518951,
      "rewards/margins": 4.94011926651001,
      "rewards/rejected": -4.767142295837402,
      "step": 405,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.6936835922900315,
      "grad_norm": 2.731245517730713,
      "learning_rate": 0.00015560109974222447,
      "logits/chosen": -0.9798430800437927,
      "logits/rejected": -0.9921943545341492,
      "logps/chosen": -3.7934410572052,
      "logps/rejected": -63.85685729980469,
      "loss": 0.24544358253479004,
      "memory(GiB)": 46.38,
      "nll_loss": 0.16812103986740112,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2368481308221817,
      "rewards/margins": 5.366214752197266,
      "rewards/rejected": -5.129366397857666,
      "step": 406,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6953921725666079,
      "grad_norm": 10.828792572021484,
      "learning_rate": 0.00015536584508376257,
      "logits/chosen": -0.9759471416473389,
      "logits/rejected": -0.9824516773223877,
      "logps/chosen": -11.430663108825684,
      "logps/rejected": -57.235267639160156,
      "loss": 0.443503201007843,
      "memory(GiB)": 46.38,
      "nll_loss": 0.39813947677612305,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.5037965774536133,
      "rewards/margins": 5.1047749519348145,
      "rewards/rejected": -4.600978374481201,
      "step": 407,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6971007528431844,
      "grad_norm": 1.4876177310943604,
      "learning_rate": 0.00015513014772147873,
      "logits/chosen": -0.9817855358123779,
      "logits/rejected": -0.9914953708648682,
      "logps/chosen": -5.813037395477295,
      "logps/rejected": -57.59131622314453,
      "loss": 0.3915078043937683,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27610617876052856,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.22342810034751892,
      "rewards/margins": 4.686631679534912,
      "rewards/rejected": -4.463203430175781,
      "step": 408,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.6988093331197608,
      "grad_norm": 13.403142929077148,
      "learning_rate": 0.00015489400954000283,
      "logits/chosen": -1.0027947425842285,
      "logits/rejected": -1.0143249034881592,
      "logps/chosen": -5.1272077560424805,
      "logps/rejected": -63.56679153442383,
      "loss": 0.40959176421165466,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2796730101108551,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26973795890808105,
      "rewards/margins": 5.1256914138793945,
      "rewards/rejected": -4.855953216552734,
      "step": 409,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7005179133963372,
      "grad_norm": 3.3234660625457764,
      "learning_rate": 0.0001546574324274894,
      "logits/chosen": -0.979751706123352,
      "logits/rejected": -0.9863253831863403,
      "logps/chosen": -10.95775032043457,
      "logps/rejected": -55.20626449584961,
      "loss": 0.4248949885368347,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33719080686569214,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26248618960380554,
      "rewards/margins": 4.562793731689453,
      "rewards/rejected": -4.300307273864746,
      "step": 410,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7022264936729137,
      "grad_norm": 10.338562965393066,
      "learning_rate": 0.00015442041827560274,
      "logits/chosen": -0.9993882179260254,
      "logits/rejected": -1.0024431943893433,
      "logps/chosen": -10.946168899536133,
      "logps/rejected": -48.0474739074707,
      "loss": 0.5547603368759155,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3820224404335022,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.04616387560963631,
      "rewards/margins": 3.627100944519043,
      "rewards/rejected": -3.673264503479004,
      "step": 411,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7039350739494901,
      "grad_norm": 14.83209228515625,
      "learning_rate": 0.00015418296897950172,
      "logits/chosen": -0.9833246469497681,
      "logits/rejected": -0.9883068799972534,
      "logps/chosen": -7.833256721496582,
      "logps/rejected": -42.49120330810547,
      "loss": 1.015760898590088,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5643466711044312,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.09800499677658081,
      "rewards/margins": 3.2539851665496826,
      "rewards/rejected": -3.351989984512329,
      "step": 412,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7056436542260666,
      "grad_norm": 11.93245792388916,
      "learning_rate": 0.00015394508643782457,
      "logits/chosen": -0.9747318029403687,
      "logits/rejected": -0.9825782775878906,
      "logps/chosen": -5.181483268737793,
      "logps/rejected": -56.730098724365234,
      "loss": 0.3347037732601166,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2391006201505661,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.12337135523557663,
      "rewards/margins": 4.270166397094727,
      "rewards/rejected": -4.14679479598999,
      "step": 413,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.707352234502643,
      "grad_norm": 5.792206287384033,
      "learning_rate": 0.00015370677255267375,
      "logits/chosen": -0.9749188423156738,
      "logits/rejected": -0.981926679611206,
      "logps/chosen": -7.287938594818115,
      "logps/rejected": -42.952945709228516,
      "loss": 0.5728598833084106,
      "memory(GiB)": 46.38,
      "nll_loss": 0.35760948061943054,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.19243726134300232,
      "rewards/margins": 3.361198663711548,
      "rewards/rejected": -3.168761730194092,
      "step": 414,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7090608147792193,
      "grad_norm": 8.534125328063965,
      "learning_rate": 0.00015346802922960073,
      "logits/chosen": -0.9716799259185791,
      "logits/rejected": -0.9807620048522949,
      "logps/chosen": -6.168636798858643,
      "logps/rejected": -51.3089599609375,
      "loss": 0.37995481491088867,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25333335995674133,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2552017271518707,
      "rewards/margins": 4.150843620300293,
      "rewards/rejected": -3.895641803741455,
      "step": 415,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7107693950557958,
      "grad_norm": 4.742763042449951,
      "learning_rate": 0.0001532288583775908,
      "logits/chosen": -0.9328129291534424,
      "logits/rejected": -0.9346539974212646,
      "logps/chosen": -9.0513277053833,
      "logps/rejected": -48.2960205078125,
      "loss": 0.6364814639091492,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4441700279712677,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05291716381907463,
      "rewards/margins": 3.499514579772949,
      "rewards/rejected": -3.4465973377227783,
      "step": 416,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7124779753323722,
      "grad_norm": 2.0339720249176025,
      "learning_rate": 0.00015298926190904767,
      "logits/chosen": -0.9535510540008545,
      "logits/rejected": -0.9562236070632935,
      "logps/chosen": -7.311025142669678,
      "logps/rejected": -40.378997802734375,
      "loss": 0.3866935670375824,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2545420527458191,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10068012773990631,
      "rewards/margins": 3.1546318531036377,
      "rewards/rejected": -3.0539519786834717,
      "step": 417,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7141865556089487,
      "grad_norm": 2.8064582347869873,
      "learning_rate": 0.00015274924173977833,
      "logits/chosen": -0.9323776960372925,
      "logits/rejected": -0.9415677189826965,
      "logps/chosen": -10.439836502075195,
      "logps/rejected": -51.31183624267578,
      "loss": 0.49138689041137695,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3278771638870239,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14516454935073853,
      "rewards/margins": 3.4552648067474365,
      "rewards/rejected": -3.3101003170013428,
      "step": 418,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7158951358855251,
      "grad_norm": 30.760719299316406,
      "learning_rate": 0.00015250879978897768,
      "logits/chosen": -0.8973679542541504,
      "logits/rejected": -0.9000917077064514,
      "logps/chosen": -6.651916980743408,
      "logps/rejected": -48.029136657714844,
      "loss": 0.47615718841552734,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32260072231292725,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26377832889556885,
      "rewards/margins": 3.9601027965545654,
      "rewards/rejected": -3.696324110031128,
      "step": 419,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7176037161621015,
      "grad_norm": 10.382326126098633,
      "learning_rate": 0.00015226793797921314,
      "logits/chosen": -0.890352189540863,
      "logits/rejected": -0.8952972292900085,
      "logps/chosen": -7.264674663543701,
      "logps/rejected": -47.71007537841797,
      "loss": 0.784246563911438,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5176042914390564,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.09244190156459808,
      "rewards/margins": 3.719733953475952,
      "rewards/rejected": -3.6272919178009033,
      "step": 420,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.719312296438678,
      "grad_norm": 4.769463539123535,
      "learning_rate": 0.00015202665823640926,
      "logits/chosen": -0.931709349155426,
      "logits/rejected": -0.933698832988739,
      "logps/chosen": -9.231295585632324,
      "logps/rejected": -45.5181770324707,
      "loss": 0.5573751926422119,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41531097888946533,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23560898005962372,
      "rewards/margins": 3.738494634628296,
      "rewards/rejected": -3.5028858184814453,
      "step": 421,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7210208767152544,
      "grad_norm": 1.436806082725525,
      "learning_rate": 0.00015178496248983254,
      "logits/chosen": -0.9031078219413757,
      "logits/rejected": -0.9134511947631836,
      "logps/chosen": -3.4011170864105225,
      "logps/rejected": -54.307594299316406,
      "loss": 0.2324237823486328,
      "memory(GiB)": 46.38,
      "nll_loss": 0.15369947254657745,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2844723165035248,
      "rewards/margins": 4.3341779708862305,
      "rewards/rejected": -4.049705982208252,
      "step": 422,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7227294569918309,
      "grad_norm": 10.88052749633789,
      "learning_rate": 0.0001515428526720757,
      "logits/chosen": -0.9290229082107544,
      "logits/rejected": -0.9370309114456177,
      "logps/chosen": -7.160928726196289,
      "logps/rejected": -56.035064697265625,
      "loss": 0.5756080746650696,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23942002654075623,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.17351604998111725,
      "rewards/margins": 4.159854888916016,
      "rewards/rejected": -3.9863390922546387,
      "step": 423,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7244380372684073,
      "grad_norm": 3.106889247894287,
      "learning_rate": 0.00015130033071904234,
      "logits/chosen": -0.9125574827194214,
      "logits/rejected": -0.9171683192253113,
      "logps/chosen": -6.642484664916992,
      "logps/rejected": -39.50868606567383,
      "loss": 0.45211318135261536,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2647590637207031,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2375515103340149,
      "rewards/margins": 3.026946544647217,
      "rewards/rejected": -2.7893946170806885,
      "step": 424,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7261466175449837,
      "grad_norm": 5.539947032928467,
      "learning_rate": 0.0001510573985699316,
      "logits/chosen": -0.919526219367981,
      "logits/rejected": -0.9228727221488953,
      "logps/chosen": -8.217347145080566,
      "logps/rejected": -44.52571487426758,
      "loss": 0.4832554757595062,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3108062148094177,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.25781702995300293,
      "rewards/margins": 3.527329683303833,
      "rewards/rejected": -3.269512176513672,
      "step": 425,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7278551978215602,
      "grad_norm": 4.5636467933654785,
      "learning_rate": 0.00015081405816722238,
      "logits/chosen": -0.9408663511276245,
      "logits/rejected": -0.9452791213989258,
      "logps/chosen": -5.501557350158691,
      "logps/rejected": -50.89080047607422,
      "loss": 0.34053143858909607,
      "memory(GiB)": 46.38,
      "nll_loss": 0.21484142541885376,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22867700457572937,
      "rewards/margins": 4.348913192749023,
      "rewards/rejected": -4.120236396789551,
      "step": 426,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7295637780981366,
      "grad_norm": 12.042389869689941,
      "learning_rate": 0.00015057031145665817,
      "logits/chosen": -0.8962785601615906,
      "logits/rejected": -0.9058489799499512,
      "logps/chosen": -10.074942588806152,
      "logps/rejected": -53.27968978881836,
      "loss": 0.6731526851654053,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36994946002960205,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.04239245876669884,
      "rewards/margins": 3.876527786254883,
      "rewards/rejected": -3.8341352939605713,
      "step": 427,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.731272358374713,
      "grad_norm": 1.6667451858520508,
      "learning_rate": 0.00015032616038723108,
      "logits/chosen": -0.8896497488021851,
      "logits/rejected": -0.8974912166595459,
      "logps/chosen": -3.299168586730957,
      "logps/rejected": -47.701114654541016,
      "loss": 0.3469572961330414,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2347521036863327,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26284629106521606,
      "rewards/margins": 4.011994361877441,
      "rewards/rejected": -3.7491486072540283,
      "step": 428,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7329809386512894,
      "grad_norm": 2.9534337520599365,
      "learning_rate": 0.0001500816069111666,
      "logits/chosen": -0.9028964638710022,
      "logits/rejected": -0.9068097472190857,
      "logps/chosen": -5.847749710083008,
      "logps/rejected": -46.586639404296875,
      "loss": 0.3202175199985504,
      "memory(GiB)": 46.38,
      "nll_loss": 0.22642484307289124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4422529637813568,
      "rewards/margins": 4.120500564575195,
      "rewards/rejected": -3.6782476902008057,
      "step": 429,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7346895189278658,
      "grad_norm": 4.043935775756836,
      "learning_rate": 0.0001498366529839078,
      "logits/chosen": -0.9180135726928711,
      "logits/rejected": -0.9261845350265503,
      "logps/chosen": -5.106454372406006,
      "logps/rejected": -53.75185012817383,
      "loss": 0.3298344016075134,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20516395568847656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.28787562251091003,
      "rewards/margins": 4.304614543914795,
      "rewards/rejected": -4.0167388916015625,
      "step": 430,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7363980992044423,
      "grad_norm": 4.341893196105957,
      "learning_rate": 0.00014959130056409976,
      "logits/chosen": -0.8813931345939636,
      "logits/rejected": -0.8917829394340515,
      "logps/chosen": -7.157702445983887,
      "logps/rejected": -55.10890579223633,
      "loss": 0.3822009861469269,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2754322290420532,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.19388091564178467,
      "rewards/margins": 4.615251541137695,
      "rewards/rejected": -4.421370506286621,
      "step": 431,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7381066794810187,
      "grad_norm": 2.139347553253174,
      "learning_rate": 0.00014934555161357388,
      "logits/chosen": -0.8547463417053223,
      "logits/rejected": -0.8581247329711914,
      "logps/chosen": -6.973649978637695,
      "logps/rejected": -53.74665069580078,
      "loss": 0.41325563192367554,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30909353494644165,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2735614776611328,
      "rewards/margins": 4.694948673248291,
      "rewards/rejected": -4.421387672424316,
      "step": 432,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7398152597575952,
      "grad_norm": 7.171928405761719,
      "learning_rate": 0.00014909940809733222,
      "logits/chosen": -0.8443334102630615,
      "logits/rejected": -0.8523035049438477,
      "logps/chosen": -5.307162284851074,
      "logps/rejected": -57.7542724609375,
      "loss": 0.3331158459186554,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25283074378967285,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2686920464038849,
      "rewards/margins": 4.889005661010742,
      "rewards/rejected": -4.62031364440918,
      "step": 433,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7415238400341716,
      "grad_norm": 15.78150749206543,
      "learning_rate": 0.0001488528719835318,
      "logits/chosen": -0.8613842725753784,
      "logits/rejected": -0.8672946691513062,
      "logps/chosen": -6.6079325675964355,
      "logps/rejected": -52.41611099243164,
      "loss": 0.46699267625808716,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31588825583457947,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.16692501306533813,
      "rewards/margins": 4.090147495269775,
      "rewards/rejected": -3.923222541809082,
      "step": 434,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.743232420310748,
      "grad_norm": 12.826034545898438,
      "learning_rate": 0.0001486059452434688,
      "logits/chosen": -0.8806869387626648,
      "logits/rejected": -0.8901012539863586,
      "logps/chosen": -7.552596569061279,
      "logps/rejected": -70.57582092285156,
      "loss": 0.45508190989494324,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4114455282688141,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.13800832629203796,
      "rewards/margins": 5.377710342407227,
      "rewards/rejected": -5.239702224731445,
      "step": 435,
      "train_speed(iter/s)": 0.005449
    },
    {
      "epoch": 0.7449410005873245,
      "grad_norm": 1.5044338703155518,
      "learning_rate": 0.00014835862985156282,
      "logits/chosen": -0.860025942325592,
      "logits/rejected": -0.8694493174552917,
      "logps/chosen": -5.0620341300964355,
      "logps/rejected": -58.80875015258789,
      "loss": 0.3588765263557434,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2822641134262085,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.39522218704223633,
      "rewards/margins": 5.250241279602051,
      "rewards/rejected": -4.8550190925598145,
      "step": 436,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7466495808639009,
      "grad_norm": 2.1413912773132324,
      "learning_rate": 0.0001481109277853412,
      "logits/chosen": -0.8253340721130371,
      "logits/rejected": -0.833922266960144,
      "logps/chosen": -5.957221031188965,
      "logps/rejected": -61.24492645263672,
      "loss": 0.31010952591896057,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19529618322849274,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.13276183605194092,
      "rewards/margins": 4.956691265106201,
      "rewards/rejected": -4.823929786682129,
      "step": 437,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7483581611404774,
      "grad_norm": 1.1828274726867676,
      "learning_rate": 0.000147862841025423,
      "logits/chosen": -0.8020204901695251,
      "logits/rejected": -0.8151015639305115,
      "logps/chosen": -3.875580310821533,
      "logps/rejected": -78.29076385498047,
      "loss": 0.17695677280426025,
      "memory(GiB)": 46.38,
      "nll_loss": 0.14975884556770325,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.381283164024353,
      "rewards/margins": 6.329374313354492,
      "rewards/rejected": -5.94809103012085,
      "step": 438,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7500667414170538,
      "grad_norm": 1.681395411491394,
      "learning_rate": 0.00014761437155550342,
      "logits/chosen": -0.8565961122512817,
      "logits/rejected": -0.8610854148864746,
      "logps/chosen": -5.606441974639893,
      "logps/rejected": -60.461578369140625,
      "loss": 0.2946867346763611,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2446092963218689,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.330669641494751,
      "rewards/margins": 5.262646198272705,
      "rewards/rejected": -4.931976795196533,
      "step": 439,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7517753216936301,
      "grad_norm": 6.150371551513672,
      "learning_rate": 0.00014736552136233763,
      "logits/chosen": -0.8346319794654846,
      "logits/rejected": -0.8419724702835083,
      "logps/chosen": -5.5094499588012695,
      "logps/rejected": -62.85390090942383,
      "loss": 0.3707015812397003,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3000151515007019,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23670655488967896,
      "rewards/margins": 5.497576713562012,
      "rewards/rejected": -5.260869979858398,
      "step": 440,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7534839019702066,
      "grad_norm": 2.236659288406372,
      "learning_rate": 0.00014711629243572525,
      "logits/chosen": -0.8184827566146851,
      "logits/rejected": -0.830502986907959,
      "logps/chosen": -4.642136096954346,
      "logps/rejected": -71.67008972167969,
      "loss": 0.31446197628974915,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2493433803319931,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3098611533641815,
      "rewards/margins": 5.951913356781006,
      "rewards/rejected": -5.642052173614502,
      "step": 441,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.755192482246783,
      "grad_norm": 16.587326049804688,
      "learning_rate": 0.00014686668676849402,
      "logits/chosen": -0.9132672548294067,
      "logits/rejected": -0.9155442714691162,
      "logps/chosen": -7.242555141448975,
      "logps/rejected": -58.58038330078125,
      "loss": 0.4143051505088806,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2861784100532532,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2265302836894989,
      "rewards/margins": 5.075270652770996,
      "rewards/rejected": -4.848740100860596,
      "step": 442,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7569010625233595,
      "grad_norm": 5.171546459197998,
      "learning_rate": 0.0001466167063564843,
      "logits/chosen": -0.8864144682884216,
      "logits/rejected": -0.8918170928955078,
      "logps/chosen": -9.903635025024414,
      "logps/rejected": -65.02162170410156,
      "loss": 0.5826258063316345,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41825902462005615,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1238093376159668,
      "rewards/margins": 5.233463287353516,
      "rewards/rejected": -5.109653949737549,
      "step": 443,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7586096427999359,
      "grad_norm": 13.517253875732422,
      "learning_rate": 0.00014636635319853275,
      "logits/chosen": -0.8727518916130066,
      "logits/rejected": -0.8807681202888489,
      "logps/chosen": -5.986981391906738,
      "logps/rejected": -56.507564544677734,
      "loss": 0.33885669708251953,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2518788278102875,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3746410608291626,
      "rewards/margins": 5.073376655578613,
      "rewards/rejected": -4.69873571395874,
      "step": 444,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7603182230765123,
      "grad_norm": 2.9981749057769775,
      "learning_rate": 0.00014611562929645654,
      "logits/chosen": -0.8742988109588623,
      "logits/rejected": -0.8825361132621765,
      "logps/chosen": -6.683349132537842,
      "logps/rejected": -62.71405029296875,
      "loss": 0.5693515539169312,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3655882775783539,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.06311008334159851,
      "rewards/margins": 5.392746925354004,
      "rewards/rejected": -5.329637050628662,
      "step": 445,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7620268033530888,
      "grad_norm": 1.6985270977020264,
      "learning_rate": 0.00014586453665503738,
      "logits/chosen": -0.907203733921051,
      "logits/rejected": -0.9205116629600525,
      "logps/chosen": -8.72842788696289,
      "logps/rejected": -76.55171203613281,
      "loss": 0.29438042640686035,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2625132203102112,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2732579708099365,
      "rewards/margins": 6.08156681060791,
      "rewards/rejected": -5.8083086013793945,
      "step": 446,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7637353836296652,
      "grad_norm": 9.158636093139648,
      "learning_rate": 0.00014561307728200526,
      "logits/chosen": -0.8190292119979858,
      "logits/rejected": -0.8243274092674255,
      "logps/chosen": -6.578858852386475,
      "logps/rejected": -54.941017150878906,
      "loss": 0.4112153947353363,
      "memory(GiB)": 46.38,
      "nll_loss": 0.310488760471344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.29622721672058105,
      "rewards/margins": 4.485001087188721,
      "rewards/rejected": -4.188774108886719,
      "step": 447,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7654439639062417,
      "grad_norm": 4.237267971038818,
      "learning_rate": 0.00014536125318802268,
      "logits/chosen": -0.8852105140686035,
      "logits/rejected": -0.8948113322257996,
      "logps/chosen": -7.432953834533691,
      "logps/rejected": -63.555789947509766,
      "loss": 0.34338587522506714,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30481523275375366,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4994341731071472,
      "rewards/margins": 5.483311653137207,
      "rewards/rejected": -4.983877658843994,
      "step": 448,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7671525441828181,
      "grad_norm": 8.01525592803955,
      "learning_rate": 0.00014510906638666833,
      "logits/chosen": -0.9222946763038635,
      "logits/rejected": -0.9368487000465393,
      "logps/chosen": -5.6248931884765625,
      "logps/rejected": -57.49897384643555,
      "loss": 0.43439823389053345,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27233919501304626,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.14373230934143066,
      "rewards/margins": 4.655035972595215,
      "rewards/rejected": -4.511303424835205,
      "step": 449,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7688611244593945,
      "grad_norm": 3.7286040782928467,
      "learning_rate": 0.00014485651889442123,
      "logits/chosen": -0.9406851530075073,
      "logits/rejected": -0.9473151564598083,
      "logps/chosen": -5.833679676055908,
      "logps/rejected": -61.25674057006836,
      "loss": 0.30055931210517883,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2684103846549988,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3485378324985504,
      "rewards/margins": 5.259521961212158,
      "rewards/rejected": -4.910984039306641,
      "step": 450,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.770569704735971,
      "grad_norm": 6.307794094085693,
      "learning_rate": 0.0001446036127306443,
      "logits/chosen": -0.9097316265106201,
      "logits/rejected": -0.9136959910392761,
      "logps/chosen": -9.636119842529297,
      "logps/rejected": -60.580875396728516,
      "loss": 0.41991665959358215,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3687485158443451,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.23546262085437775,
      "rewards/margins": 5.274323463439941,
      "rewards/rejected": -5.03886079788208,
      "step": 451,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7722782850125474,
      "grad_norm": 7.95340633392334,
      "learning_rate": 0.0001443503499175686,
      "logits/chosen": -0.9337123036384583,
      "logits/rejected": -0.9448880553245544,
      "logps/chosen": -10.418018341064453,
      "logps/rejected": -57.76142120361328,
      "loss": 1.1063024997711182,
      "memory(GiB)": 46.38,
      "nll_loss": 0.7773098349571228,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11964107304811478,
      "rewards/margins": 4.844240188598633,
      "rewards/rejected": -4.724598407745361,
      "step": 452,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7739868652891239,
      "grad_norm": 26.539628982543945,
      "learning_rate": 0.00014409673248027676,
      "logits/chosen": -0.897848904132843,
      "logits/rejected": -0.9111236333847046,
      "logps/chosen": -7.773808002471924,
      "logps/rejected": -56.539024353027344,
      "loss": 0.34956812858581543,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30711376667022705,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.540047287940979,
      "rewards/margins": 4.916183948516846,
      "rewards/rejected": -4.376136779785156,
      "step": 453,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7756954455657002,
      "grad_norm": 14.550936698913574,
      "learning_rate": 0.00014384276244668704,
      "logits/chosen": -0.9153473377227783,
      "logits/rejected": -0.9219194054603577,
      "logps/chosen": -6.920995712280273,
      "logps/rejected": -58.91932678222656,
      "loss": 0.39858752489089966,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30973657965660095,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2662816643714905,
      "rewards/margins": 5.184060096740723,
      "rewards/rejected": -4.917778491973877,
      "step": 454,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7774040258422766,
      "grad_norm": 20.295856475830078,
      "learning_rate": 0.00014358844184753712,
      "logits/chosen": -0.9467324018478394,
      "logits/rejected": -0.9530484676361084,
      "logps/chosen": -6.200811386108398,
      "logps/rejected": -51.07537078857422,
      "loss": 0.4701750874519348,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34591811895370483,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2089444100856781,
      "rewards/margins": 4.312093734741211,
      "rewards/rejected": -4.103148937225342,
      "step": 455,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7791126061188531,
      "grad_norm": 1.5724072456359863,
      "learning_rate": 0.00014333377271636766,
      "logits/chosen": -0.9356569647789001,
      "logits/rejected": -0.9412778615951538,
      "logps/chosen": -5.570844650268555,
      "logps/rejected": -57.72052764892578,
      "loss": 0.3009592294692993,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2671731114387512,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2683850824832916,
      "rewards/margins": 5.20340633392334,
      "rewards/rejected": -4.93502140045166,
      "step": 456,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7808211863954295,
      "grad_norm": 30.807010650634766,
      "learning_rate": 0.00014307875708950636,
      "logits/chosen": -0.8859197497367859,
      "logits/rejected": -0.8948801159858704,
      "logps/chosen": -7.5421576499938965,
      "logps/rejected": -57.7672119140625,
      "loss": 0.5995492935180664,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4698564410209656,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11614438891410828,
      "rewards/margins": 4.431459426879883,
      "rewards/rejected": -4.315314292907715,
      "step": 457,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.782529766672006,
      "grad_norm": 1.4951168298721313,
      "learning_rate": 0.00014282339700605125,
      "logits/chosen": -0.9123712778091431,
      "logits/rejected": -0.9234874844551086,
      "logps/chosen": -8.449228286743164,
      "logps/rejected": -60.451656341552734,
      "loss": 0.4394095838069916,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4088311791419983,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.49215221405029297,
      "rewards/margins": 5.459771156311035,
      "rewards/rejected": -4.9676194190979,
      "step": 458,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7842383469485824,
      "grad_norm": 14.627826690673828,
      "learning_rate": 0.0001425676945078548,
      "logits/chosen": -0.9516683220863342,
      "logits/rejected": -0.9583043456077576,
      "logps/chosen": -8.45793628692627,
      "logps/rejected": -53.02428436279297,
      "loss": 0.8185063004493713,
      "memory(GiB)": 46.38,
      "nll_loss": 0.601269543170929,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.022628281265497208,
      "rewards/margins": 4.273806571960449,
      "rewards/rejected": -4.2964348793029785,
      "step": 459,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7859469272251588,
      "grad_norm": 3.7965071201324463,
      "learning_rate": 0.00014231165163950744,
      "logits/chosen": -0.8973619937896729,
      "logits/rejected": -0.9101439714431763,
      "logps/chosen": -7.475704669952393,
      "logps/rejected": -59.00372314453125,
      "loss": 0.8011679649353027,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5742935538291931,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16849175095558167,
      "rewards/margins": 4.538441181182861,
      "rewards/rejected": -4.3699493408203125,
      "step": 460,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.7876555075017353,
      "grad_norm": 5.5856242179870605,
      "learning_rate": 0.000142055270448321,
      "logits/chosen": -0.8714302778244019,
      "logits/rejected": -0.8758531212806702,
      "logps/chosen": -7.770807266235352,
      "logps/rejected": -48.69560623168945,
      "loss": 0.44547683000564575,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3522818684577942,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.17969879508018494,
      "rewards/margins": 4.13702917098999,
      "rewards/rejected": -3.9573302268981934,
      "step": 461,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7893640877783117,
      "grad_norm": 16.880699157714844,
      "learning_rate": 0.00014179855298431277,
      "logits/chosen": -0.873023509979248,
      "logits/rejected": -0.8793730735778809,
      "logps/chosen": -7.280762672424316,
      "logps/rejected": -54.3899040222168,
      "loss": 0.4869714379310608,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36446613073349,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1354772299528122,
      "rewards/margins": 4.172247409820557,
      "rewards/rejected": -4.036769866943359,
      "step": 462,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7910726680548882,
      "grad_norm": 2.617577314376831,
      "learning_rate": 0.00014154150130018866,
      "logits/chosen": -0.8602393865585327,
      "logits/rejected": -0.8721433281898499,
      "logps/chosen": -6.432670593261719,
      "logps/rejected": -58.89222717285156,
      "loss": 0.3192591965198517,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24738143384456635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19203299283981323,
      "rewards/margins": 4.471967697143555,
      "rewards/rejected": -4.279934883117676,
      "step": 463,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7927812483314646,
      "grad_norm": 14.798487663269043,
      "learning_rate": 0.00014128411745132713,
      "logits/chosen": -0.873883843421936,
      "logits/rejected": -0.8781724572181702,
      "logps/chosen": -10.722978591918945,
      "logps/rejected": -45.717071533203125,
      "loss": 0.7172985672950745,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49196088314056396,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2104509323835373,
      "rewards/margins": 3.9081997871398926,
      "rewards/rejected": -3.697748899459839,
      "step": 464,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.794489828608041,
      "grad_norm": 3.1615259647369385,
      "learning_rate": 0.00014102640349576256,
      "logits/chosen": -0.8448261618614197,
      "logits/rejected": -0.8480522036552429,
      "logps/chosen": -8.05765151977539,
      "logps/rejected": -50.27508544921875,
      "loss": 0.41961443424224854,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3122217357158661,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26587533950805664,
      "rewards/margins": 4.0656208992004395,
      "rewards/rejected": -3.799746036529541,
      "step": 465,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7961984088846175,
      "grad_norm": 1.8068488836288452,
      "learning_rate": 0.00014076836149416887,
      "logits/chosen": -0.8060347437858582,
      "logits/rejected": -0.8118148446083069,
      "logps/chosen": -9.203452110290527,
      "logps/rejected": -52.506805419921875,
      "loss": 0.42939236760139465,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3758169710636139,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2894461750984192,
      "rewards/margins": 4.457134246826172,
      "rewards/rejected": -4.167688369750977,
      "step": 466,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7979069891611938,
      "grad_norm": 2.6210246086120605,
      "learning_rate": 0.00014050999350984306,
      "logits/chosen": -0.8480339646339417,
      "logits/rejected": -0.85347580909729,
      "logps/chosen": -7.270939826965332,
      "logps/rejected": -49.5122184753418,
      "loss": 0.5322786569595337,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32996895909309387,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0504617877304554,
      "rewards/margins": 3.751173257827759,
      "rewards/rejected": -3.7007110118865967,
      "step": 467,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.7996155694377703,
      "grad_norm": 14.975828170776367,
      "learning_rate": 0.00014025130160868864,
      "logits/chosen": -0.8458459973335266,
      "logits/rejected": -0.8523356914520264,
      "logps/chosen": -9.00961685180664,
      "logps/rejected": -45.268985748291016,
      "loss": 0.5993744730949402,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4293050467967987,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.40115952491760254,
      "rewards/margins": 3.7434749603271484,
      "rewards/rejected": -3.342315196990967,
      "step": 468,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.8013241497143467,
      "grad_norm": 4.5701446533203125,
      "learning_rate": 0.00013999228785919915,
      "logits/chosen": -0.8248124122619629,
      "logits/rejected": -0.8288387060165405,
      "logps/chosen": -6.494216442108154,
      "logps/rejected": -42.423057556152344,
      "loss": 0.6713830232620239,
      "memory(GiB)": 46.38,
      "nll_loss": 0.48319515585899353,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3284499943256378,
      "rewards/margins": 3.4159247875213623,
      "rewards/rejected": -3.087474822998047,
      "step": 469,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8030327299909231,
      "grad_norm": 1.6908752918243408,
      "learning_rate": 0.0001397329543324416,
      "logits/chosen": -0.8458975553512573,
      "logits/rejected": -0.8548964858055115,
      "logps/chosen": -6.597393989562988,
      "logps/rejected": -52.00492858886719,
      "loss": 0.37268275022506714,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2903206944465637,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14557278156280518,
      "rewards/margins": 3.8265743255615234,
      "rewards/rejected": -3.681001663208008,
      "step": 470,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8047413102674996,
      "grad_norm": 12.578397750854492,
      "learning_rate": 0.00013947330310204,
      "logits/chosen": -0.840053915977478,
      "logits/rejected": -0.8506962656974792,
      "logps/chosen": -5.983945846557617,
      "logps/rejected": -51.502113342285156,
      "loss": 0.4621216356754303,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3134876489639282,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2717393934726715,
      "rewards/margins": 3.639366865158081,
      "rewards/rejected": -3.3676278591156006,
      "step": 471,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.806449890544076,
      "grad_norm": 3.028996467590332,
      "learning_rate": 0.00013921333624415864,
      "logits/chosen": -0.8801653981208801,
      "logits/rejected": -0.885321855545044,
      "logps/chosen": -6.084489822387695,
      "logps/rejected": -49.61780548095703,
      "loss": 0.3569665253162384,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2814903259277344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3482724130153656,
      "rewards/margins": 3.9839084148406982,
      "rewards/rejected": -3.6356358528137207,
      "step": 472,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8081584708206525,
      "grad_norm": 1.3438864946365356,
      "learning_rate": 0.00013895305583748563,
      "logits/chosen": -0.855207085609436,
      "logits/rejected": -0.8606567978858948,
      "logps/chosen": -8.977983474731445,
      "logps/rejected": -53.27305603027344,
      "loss": 0.314006507396698,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2651103138923645,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2757810354232788,
      "rewards/margins": 4.300058841705322,
      "rewards/rejected": -4.024277687072754,
      "step": 473,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.8098670510972289,
      "grad_norm": 3.45967435836792,
      "learning_rate": 0.00013869246396321612,
      "logits/chosen": -0.9474461078643799,
      "logits/rejected": -0.9534441232681274,
      "logps/chosen": -6.840535640716553,
      "logps/rejected": -40.288883209228516,
      "loss": 0.46610358357429504,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2519730031490326,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.07268978655338287,
      "rewards/margins": 3.1671252250671387,
      "rewards/rejected": -3.094435453414917,
      "step": 474,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8115756313738053,
      "grad_norm": 37.44761657714844,
      "learning_rate": 0.00013843156270503578,
      "logits/chosen": -0.9077054262161255,
      "logits/rejected": -0.9159120321273804,
      "logps/chosen": -6.894405841827393,
      "logps/rejected": -51.91642379760742,
      "loss": 0.5031276345252991,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40508726239204407,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.28744038939476013,
      "rewards/margins": 4.0617499351501465,
      "rewards/rejected": -3.7743093967437744,
      "step": 475,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8132842116503818,
      "grad_norm": 4.589206218719482,
      "learning_rate": 0.00013817035414910418,
      "logits/chosen": -0.9251917600631714,
      "logits/rejected": -0.9348454475402832,
      "logps/chosen": -4.828737735748291,
      "logps/rejected": -51.33116149902344,
      "loss": 0.3270384669303894,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2073853760957718,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2917906641960144,
      "rewards/margins": 4.043920993804932,
      "rewards/rejected": -3.7521302700042725,
      "step": 476,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8149927919269582,
      "grad_norm": 6.489503860473633,
      "learning_rate": 0.00013790884038403795,
      "logits/chosen": -0.9607729315757751,
      "logits/rejected": -0.9636399745941162,
      "logps/chosen": -9.576072692871094,
      "logps/rejected": -48.74690246582031,
      "loss": 0.5957178473472595,
      "memory(GiB)": 46.38,
      "nll_loss": 0.43875014781951904,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.08807779103517532,
      "rewards/margins": 3.889638662338257,
      "rewards/rejected": -3.801560878753662,
      "step": 477,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.8167013722035347,
      "grad_norm": 2.334988594055176,
      "learning_rate": 0.00013764702350089415,
      "logits/chosen": -0.9154888391494751,
      "logits/rejected": -0.9213120341300964,
      "logps/chosen": -5.786940097808838,
      "logps/rejected": -50.51441955566406,
      "loss": 0.30048811435699463,
      "memory(GiB)": 46.38,
      "nll_loss": 0.22780688107013702,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.36729320883750916,
      "rewards/margins": 4.216024398803711,
      "rewards/rejected": -3.84873104095459,
      "step": 478,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8184099524801111,
      "grad_norm": 1.242850422859192,
      "learning_rate": 0.00013738490559315366,
      "logits/chosen": -0.9769160151481628,
      "logits/rejected": -0.9855915307998657,
      "logps/chosen": -4.714077472686768,
      "logps/rejected": -57.93135070800781,
      "loss": 0.21795281767845154,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17518356442451477,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21144519746303558,
      "rewards/margins": 4.761663436889648,
      "rewards/rejected": -4.550218105316162,
      "step": 479,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8201185327566874,
      "grad_norm": 3.3494644165039062,
      "learning_rate": 0.00013712248875670424,
      "logits/chosen": -0.9684186577796936,
      "logits/rejected": -0.9767059087753296,
      "logps/chosen": -9.090254783630371,
      "logps/rejected": -48.01445388793945,
      "loss": 0.484287291765213,
      "memory(GiB)": 46.38,
      "nll_loss": 0.362609326839447,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24491199851036072,
      "rewards/margins": 3.751887321472168,
      "rewards/rejected": -3.5069754123687744,
      "step": 480,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.821827113033264,
      "grad_norm": 13.496355056762695,
      "learning_rate": 0.00013685977508982392,
      "logits/chosen": -1.0050654411315918,
      "logits/rejected": -1.0114392042160034,
      "logps/chosen": -7.333817958831787,
      "logps/rejected": -49.09504318237305,
      "loss": 0.9122673273086548,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5497475266456604,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.06162791699171066,
      "rewards/margins": 3.880805730819702,
      "rewards/rejected": -3.9424338340759277,
      "step": 481,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8235356933098403,
      "grad_norm": 6.231888294219971,
      "learning_rate": 0.0001365967666931642,
      "logits/chosen": -0.950136661529541,
      "logits/rejected": -0.9556779265403748,
      "logps/chosen": -5.013396739959717,
      "logps/rejected": -53.2423210144043,
      "loss": 0.3058711588382721,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2658306062221527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2847394347190857,
      "rewards/margins": 4.522904396057129,
      "rewards/rejected": -4.238165378570557,
      "step": 482,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8252442735864168,
      "grad_norm": 2.108607530593872,
      "learning_rate": 0.00013633346566973323,
      "logits/chosen": -0.946274995803833,
      "logits/rejected": -0.9529342651367188,
      "logps/chosen": -5.831514835357666,
      "logps/rejected": -49.31201934814453,
      "loss": 0.31166088581085205,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18263982236385345,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26241183280944824,
      "rewards/margins": 3.728137969970703,
      "rewards/rejected": -3.465726137161255,
      "step": 483,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8269528538629932,
      "grad_norm": 1.4421031475067139,
      "learning_rate": 0.00013606987412487893,
      "logits/chosen": -0.961552083492279,
      "logits/rejected": -0.9671511650085449,
      "logps/chosen": -7.4079790115356445,
      "logps/rejected": -52.183441162109375,
      "loss": 0.3153613805770874,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27535566687583923,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3105275332927704,
      "rewards/margins": 4.4759321212768555,
      "rewards/rejected": -4.165404796600342,
      "step": 484,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8286614341395696,
      "grad_norm": 6.844588756561279,
      "learning_rate": 0.0001358059941662723,
      "logits/chosen": -0.9825448989868164,
      "logits/rejected": -0.9879649877548218,
      "logps/chosen": -9.175311088562012,
      "logps/rejected": -50.80815887451172,
      "loss": 0.8173354864120483,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6075851321220398,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.03993527963757515,
      "rewards/margins": 3.653517961502075,
      "rewards/rejected": -3.693453311920166,
      "step": 485,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8303700144161461,
      "grad_norm": 0.8626744151115417,
      "learning_rate": 0.00013554182790389047,
      "logits/chosen": -0.9753793478012085,
      "logits/rejected": -0.9814032912254333,
      "logps/chosen": -7.666321754455566,
      "logps/rejected": -56.26683807373047,
      "loss": 0.32768967747688293,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2923482060432434,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2374456822872162,
      "rewards/margins": 4.816155433654785,
      "rewards/rejected": -4.578710079193115,
      "step": 486,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8320785946927225,
      "grad_norm": 6.582581996917725,
      "learning_rate": 0.0001352773774499998,
      "logits/chosen": -1.0147960186004639,
      "logits/rejected": -1.020078420639038,
      "logps/chosen": -6.684427261352539,
      "logps/rejected": -44.743377685546875,
      "loss": 0.6805826425552368,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4731958508491516,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.167913019657135,
      "rewards/margins": 3.6935548782348633,
      "rewards/rejected": -3.525641918182373,
      "step": 487,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.833787174969299,
      "grad_norm": 15.331477165222168,
      "learning_rate": 0.00013501264491913906,
      "logits/chosen": -0.9626951217651367,
      "logits/rejected": -0.9696282148361206,
      "logps/chosen": -5.262228488922119,
      "logps/rejected": -51.761619567871094,
      "loss": 0.5132107734680176,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3812980353832245,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.11689464002847672,
      "rewards/margins": 4.251979351043701,
      "rewards/rejected": -4.135085105895996,
      "step": 488,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8354957552458754,
      "grad_norm": 7.233061790466309,
      "learning_rate": 0.00013474763242810257,
      "logits/chosen": -1.038739800453186,
      "logits/rejected": -1.0475245714187622,
      "logps/chosen": -6.640017509460449,
      "logps/rejected": -61.72606658935547,
      "loss": 0.3782581090927124,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34932804107666016,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3043239116668701,
      "rewards/margins": 5.470346927642822,
      "rewards/rejected": -5.166023254394531,
      "step": 489,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8372043355224518,
      "grad_norm": 1.597853422164917,
      "learning_rate": 0.00013448234209592313,
      "logits/chosen": -0.9879864454269409,
      "logits/rejected": -0.9992839694023132,
      "logps/chosen": -6.114026069641113,
      "logps/rejected": -61.805503845214844,
      "loss": 0.2753511667251587,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19614028930664062,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2151072919368744,
      "rewards/margins": 5.094541072845459,
      "rewards/rejected": -4.879433631896973,
      "step": 490,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8389129157990283,
      "grad_norm": 8.912554740905762,
      "learning_rate": 0.00013421677604385512,
      "logits/chosen": -1.0242968797683716,
      "logits/rejected": -1.0332534313201904,
      "logps/chosen": -4.353573322296143,
      "logps/rejected": -52.652828216552734,
      "loss": 0.5309911370277405,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30963706970214844,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16331471502780914,
      "rewards/margins": 4.274681568145752,
      "rewards/rejected": -4.111367225646973,
      "step": 491,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8406214960756047,
      "grad_norm": 6.16519021987915,
      "learning_rate": 0.0001339509363953577,
      "logits/chosen": -1.0380960702896118,
      "logits/rejected": -1.045656681060791,
      "logps/chosen": -8.553430557250977,
      "logps/rejected": -49.535194396972656,
      "loss": 0.4442055821418762,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33289361000061035,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.28979986906051636,
      "rewards/margins": 4.017800807952881,
      "rewards/rejected": -3.728001594543457,
      "step": 492,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8423300763521812,
      "grad_norm": 2.817603826522827,
      "learning_rate": 0.00013368482527607747,
      "logits/chosen": -1.011553168296814,
      "logits/rejected": -1.0203989744186401,
      "logps/chosen": -5.940677165985107,
      "logps/rejected": -53.4194221496582,
      "loss": 0.33772724866867065,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24142131209373474,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.22640693187713623,
      "rewards/margins": 4.491219520568848,
      "rewards/rejected": -4.264812469482422,
      "step": 493,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8440386566287575,
      "grad_norm": 6.619709014892578,
      "learning_rate": 0.00013341844481383204,
      "logits/chosen": -1.0174074172973633,
      "logits/rejected": -1.0238661766052246,
      "logps/chosen": -8.416812896728516,
      "logps/rejected": -50.30221176147461,
      "loss": 0.43706879019737244,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3408777713775635,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3022111654281616,
      "rewards/margins": 3.9875621795654297,
      "rewards/rejected": -3.6853506565093994,
      "step": 494,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8457472369053339,
      "grad_norm": 2.03102707862854,
      "learning_rate": 0.00013315179713859233,
      "logits/chosen": -1.0034698247909546,
      "logits/rejected": -1.010728359222412,
      "logps/chosen": -5.10067081451416,
      "logps/rejected": -53.246185302734375,
      "loss": 0.3427221477031708,
      "memory(GiB)": 46.38,
      "nll_loss": 0.21901173889636993,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.20335926115512848,
      "rewards/margins": 4.209983825683594,
      "rewards/rejected": -4.006624698638916,
      "step": 495,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8474558171819104,
      "grad_norm": 4.971916675567627,
      "learning_rate": 0.0001328848843824661,
      "logits/chosen": -1.0119178295135498,
      "logits/rejected": -1.0189917087554932,
      "logps/chosen": -5.981982707977295,
      "logps/rejected": -57.134925842285156,
      "loss": 0.3167141377925873,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23812633752822876,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.17970748245716095,
      "rewards/margins": 4.728905200958252,
      "rewards/rejected": -4.549197196960449,
      "step": 496,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8491643974584868,
      "grad_norm": 2.055927038192749,
      "learning_rate": 0.0001326177086796807,
      "logits/chosen": -0.9268537759780884,
      "logits/rejected": -0.933496356010437,
      "logps/chosen": -8.920307159423828,
      "logps/rejected": -59.319740295410156,
      "loss": 0.42835739254951477,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3605175018310547,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.396852970123291,
      "rewards/margins": 4.572652339935303,
      "rewards/rejected": -4.175799369812012,
      "step": 497,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8508729777350633,
      "grad_norm": 21.994491577148438,
      "learning_rate": 0.00013235027216656583,
      "logits/chosen": -0.9912822246551514,
      "logits/rejected": -0.9992126226425171,
      "logps/chosen": -4.590593338012695,
      "logps/rejected": -53.87917709350586,
      "loss": 0.3980537950992584,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26871615648269653,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2924727499485016,
      "rewards/margins": 4.3161187171936035,
      "rewards/rejected": -4.023645877838135,
      "step": 498,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8525815580116397,
      "grad_norm": 2.163431167602539,
      "learning_rate": 0.00013208257698153677,
      "logits/chosen": -0.99356609582901,
      "logits/rejected": -0.997937798500061,
      "logps/chosen": -10.751409530639648,
      "logps/rejected": -50.82136535644531,
      "loss": 0.4969412386417389,
      "memory(GiB)": 46.38,
      "nll_loss": 0.38345521688461304,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21135258674621582,
      "rewards/margins": 3.9842920303344727,
      "rewards/rejected": -3.772939682006836,
      "step": 499,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8542901382882161,
      "grad_norm": 14.22870922088623,
      "learning_rate": 0.00013181462526507708,
      "logits/chosen": -0.9459682106971741,
      "logits/rejected": -0.9542500972747803,
      "logps/chosen": -5.982848167419434,
      "logps/rejected": -53.59891128540039,
      "loss": 0.470400869846344,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3129653036594391,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.14928746223449707,
      "rewards/margins": 3.932734489440918,
      "rewards/rejected": -3.783447265625,
      "step": 500,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8559987185647926,
      "grad_norm": 4.418472766876221,
      "learning_rate": 0.00013154641915972154,
      "logits/chosen": -1.0272095203399658,
      "logits/rejected": -1.0328339338302612,
      "logps/chosen": -7.5063323974609375,
      "logps/rejected": -47.45936965942383,
      "loss": 0.4476359188556671,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33541929721832275,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.38147568702697754,
      "rewards/margins": 3.8733739852905273,
      "rewards/rejected": -3.491898536682129,
      "step": 501,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.857707298841369,
      "grad_norm": 10.935993194580078,
      "learning_rate": 0.00013127796081003904,
      "logits/chosen": -1.0141963958740234,
      "logits/rejected": -1.0185487270355225,
      "logps/chosen": -7.2610697746276855,
      "logps/rejected": -48.271949768066406,
      "loss": 0.6028414368629456,
      "memory(GiB)": 46.38,
      "nll_loss": 0.41852545738220215,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2857853174209595,
      "rewards/margins": 3.865593910217285,
      "rewards/rejected": -3.5798087120056152,
      "step": 502,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8594158791179455,
      "grad_norm": 2.554673194885254,
      "learning_rate": 0.0001310092523626154,
      "logits/chosen": -1.034954309463501,
      "logits/rejected": -1.0445373058319092,
      "logps/chosen": -8.33714485168457,
      "logps/rejected": -57.27528762817383,
      "loss": 0.3383696377277374,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27207207679748535,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3617641031742096,
      "rewards/margins": 4.716930389404297,
      "rewards/rejected": -4.355166912078857,
      "step": 503,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8611244593945219,
      "grad_norm": 1.960814356803894,
      "learning_rate": 0.00013074029596603614,
      "logits/chosen": -1.0110938549041748,
      "logits/rejected": -1.01825749874115,
      "logps/chosen": -6.963066101074219,
      "logps/rejected": -50.569862365722656,
      "loss": 0.41586294770240784,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33001551032066345,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30317986011505127,
      "rewards/margins": 4.454747200012207,
      "rewards/rejected": -4.151567459106445,
      "step": 504,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8628330396710983,
      "grad_norm": 8.382596015930176,
      "learning_rate": 0.0001304710937708695,
      "logits/chosen": -1.0239723920822144,
      "logits/rejected": -1.0359508991241455,
      "logps/chosen": -9.96099853515625,
      "logps/rejected": -68.2059555053711,
      "loss": 0.428564190864563,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3127758204936981,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27931201457977295,
      "rewards/margins": 5.435581207275391,
      "rewards/rejected": -5.156269550323486,
      "step": 505,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.8645416199476748,
      "grad_norm": 3.1605477333068848,
      "learning_rate": 0.000130201647929649,
      "logits/chosen": -1.0900601148605347,
      "logits/rejected": -1.1016480922698975,
      "logps/chosen": -8.03422737121582,
      "logps/rejected": -60.09300994873047,
      "loss": 0.4372721016407013,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3315892815589905,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2807161509990692,
      "rewards/margins": 5.148784637451172,
      "rewards/rejected": -4.868068218231201,
      "step": 506,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8662502002242511,
      "grad_norm": 6.573107719421387,
      "learning_rate": 0.0001299319605968565,
      "logits/chosen": -1.0357370376586914,
      "logits/rejected": -1.0446820259094238,
      "logps/chosen": -10.143904685974121,
      "logps/rejected": -65.32817840576172,
      "loss": 0.709911584854126,
      "memory(GiB)": 46.38,
      "nll_loss": 0.40525418519973755,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.0815616250038147,
      "rewards/margins": 5.317613124847412,
      "rewards/rejected": -5.236051559448242,
      "step": 507,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8679587805008276,
      "grad_norm": 12.627511978149414,
      "learning_rate": 0.00012966203392890464,
      "logits/chosen": -1.1144845485687256,
      "logits/rejected": -1.1247533559799194,
      "logps/chosen": -4.9375128746032715,
      "logps/rejected": -67.10102081298828,
      "loss": 0.31131261587142944,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2803378105163574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2530133128166199,
      "rewards/margins": 5.781771659851074,
      "rewards/rejected": -5.528758525848389,
      "step": 508,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.869667360777404,
      "grad_norm": 4.412936210632324,
      "learning_rate": 0.00012939187008411993,
      "logits/chosen": -1.0667747259140015,
      "logits/rejected": -1.0817590951919556,
      "logps/chosen": -7.906347751617432,
      "logps/rejected": -65.20343780517578,
      "loss": 0.5059725046157837,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4028059244155884,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.16295327246189117,
      "rewards/margins": 5.399806022644043,
      "rewards/rejected": -5.236852645874023,
      "step": 509,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8713759410539804,
      "grad_norm": 6.105998516082764,
      "learning_rate": 0.00012912147122272523,
      "logits/chosen": -1.0236029624938965,
      "logits/rejected": -1.0314514636993408,
      "logps/chosen": -10.549116134643555,
      "logps/rejected": -52.82227325439453,
      "loss": 0.7402754426002502,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36340171098709106,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.009765136986970901,
      "rewards/margins": 4.058257102966309,
      "rewards/rejected": -4.06802225112915,
      "step": 510,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8730845213305569,
      "grad_norm": 2.474003791809082,
      "learning_rate": 0.00012885083950682263,
      "logits/chosen": -1.0671751499176025,
      "logits/rejected": -1.0764691829681396,
      "logps/chosen": -7.913850784301758,
      "logps/rejected": -58.417747497558594,
      "loss": 0.38465413451194763,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30931293964385986,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27320733666419983,
      "rewards/margins": 5.048703193664551,
      "rewards/rejected": -4.775494575500488,
      "step": 511,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8747931016071333,
      "grad_norm": 6.503899574279785,
      "learning_rate": 0.00012857997710037612,
      "logits/chosen": -1.0322140455245972,
      "logits/rejected": -1.0421993732452393,
      "logps/chosen": -7.503562927246094,
      "logps/rejected": -62.2566032409668,
      "loss": 0.47043660283088684,
      "memory(GiB)": 46.38,
      "nll_loss": 0.354666531085968,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.22537949681282043,
      "rewards/margins": 5.0822296142578125,
      "rewards/rejected": -4.856850624084473,
      "step": 512,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8765016818837098,
      "grad_norm": 1.2542670965194702,
      "learning_rate": 0.00012830888616919433,
      "logits/chosen": -0.9723070859909058,
      "logits/rejected": -0.9807174205780029,
      "logps/chosen": -5.060972213745117,
      "logps/rejected": -62.85099411010742,
      "loss": 0.20766031742095947,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17876210808753967,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30771228671073914,
      "rewards/margins": 5.462485313415527,
      "rewards/rejected": -5.154772758483887,
      "step": 513,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8782102621602862,
      "grad_norm": 0.94368577003479,
      "learning_rate": 0.00012803756888091306,
      "logits/chosen": -1.0189743041992188,
      "logits/rejected": -1.0311044454574585,
      "logps/chosen": -9.014524459838867,
      "logps/rejected": -64.86925506591797,
      "loss": 0.3573024570941925,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32912033796310425,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5030632019042969,
      "rewards/margins": 5.568962097167969,
      "rewards/rejected": -5.065898895263672,
      "step": 514,
      "train_speed(iter/s)": 0.00545
    },
    {
      "epoch": 0.8799188424368626,
      "grad_norm": 12.628750801086426,
      "learning_rate": 0.00012776602740497816,
      "logits/chosen": -0.9814023971557617,
      "logits/rejected": -0.9902243614196777,
      "logps/chosen": -6.21978235244751,
      "logps/rejected": -62.96957778930664,
      "loss": 0.5857754349708557,
      "memory(GiB)": 46.38,
      "nll_loss": 0.43782201409339905,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.05701781064271927,
      "rewards/margins": 5.020473480224609,
      "rewards/rejected": -4.963456153869629,
      "step": 515,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8816274227134391,
      "grad_norm": 17.101516723632812,
      "learning_rate": 0.00012749426391262802,
      "logits/chosen": -1.0013269186019897,
      "logits/rejected": -1.0105067491531372,
      "logps/chosen": -7.749750137329102,
      "logps/rejected": -59.32505798339844,
      "loss": 0.42123281955718994,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34354791045188904,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26860707998275757,
      "rewards/margins": 4.870937824249268,
      "rewards/rejected": -4.602330684661865,
      "step": 516,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8833360029900155,
      "grad_norm": 1.1716769933700562,
      "learning_rate": 0.00012722228057687626,
      "logits/chosen": -0.9398144483566284,
      "logits/rejected": -0.9461740255355835,
      "logps/chosen": -4.985825538635254,
      "logps/rejected": -53.900360107421875,
      "loss": 0.31432977318763733,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20304182171821594,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.33839696645736694,
      "rewards/margins": 4.765352725982666,
      "rewards/rejected": -4.426955699920654,
      "step": 517,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.885044583266592,
      "grad_norm": 1.7283916473388672,
      "learning_rate": 0.00012695007957249445,
      "logits/chosen": -0.9696387648582458,
      "logits/rejected": -0.9768409132957458,
      "logps/chosen": -8.448753356933594,
      "logps/rejected": -60.238059997558594,
      "loss": 0.5001124143600464,
      "memory(GiB)": 46.38,
      "nll_loss": 0.43674081563949585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2527063190937042,
      "rewards/margins": 4.810661792755127,
      "rewards/rejected": -4.557955265045166,
      "step": 518,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8867531635431684,
      "grad_norm": 13.50886344909668,
      "learning_rate": 0.00012667766307599448,
      "logits/chosen": -0.9939947128295898,
      "logits/rejected": -1.0037935972213745,
      "logps/chosen": -6.2766876220703125,
      "logps/rejected": -60.59209442138672,
      "loss": 0.38991400599479675,
      "memory(GiB)": 46.38,
      "nll_loss": 0.33603137731552124,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4559497535228729,
      "rewards/margins": 5.023289680480957,
      "rewards/rejected": -4.5673394203186035,
      "step": 519,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8884617438197447,
      "grad_norm": 2.6633639335632324,
      "learning_rate": 0.00012640503326561144,
      "logits/chosen": -0.9777341485023499,
      "logits/rejected": -0.9868544340133667,
      "logps/chosen": -9.681154251098633,
      "logps/rejected": -59.64811325073242,
      "loss": 0.8820703029632568,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6579297780990601,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.10375190526247025,
      "rewards/margins": 4.73799467086792,
      "rewards/rejected": -4.634243011474609,
      "step": 520,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8901703240963212,
      "grad_norm": 2.603954553604126,
      "learning_rate": 0.00012613219232128608,
      "logits/chosen": -0.9925277233123779,
      "logits/rejected": -1.0005781650543213,
      "logps/chosen": -4.99767541885376,
      "logps/rejected": -51.02067184448242,
      "loss": 0.366862952709198,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2160557508468628,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.29435160756111145,
      "rewards/margins": 4.297109127044678,
      "rewards/rejected": -4.002757549285889,
      "step": 521,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8918789043728976,
      "grad_norm": 11.442673683166504,
      "learning_rate": 0.00012585914242464722,
      "logits/chosen": -1.0211143493652344,
      "logits/rejected": -1.0297880172729492,
      "logps/chosen": -5.821419715881348,
      "logps/rejected": -48.07485580444336,
      "loss": 0.6329149603843689,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3832743763923645,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1315867155790329,
      "rewards/margins": 3.7371625900268555,
      "rewards/rejected": -3.6055760383605957,
      "step": 522,
      "train_speed(iter/s)": 0.005451
    },
    {
      "epoch": 0.8935874846494741,
      "grad_norm": 0.8552331328392029,
      "learning_rate": 0.00012558588575899466,
      "logits/chosen": -1.0670050382614136,
      "logits/rejected": -1.082131266593933,
      "logps/chosen": -5.791848659515381,
      "logps/rejected": -68.00889587402344,
      "loss": 0.2681228518486023,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2114100158214569,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2899337112903595,
      "rewards/margins": 5.15542459487915,
      "rewards/rejected": -4.865490436553955,
      "step": 523,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8952960649260505,
      "grad_norm": 9.0734224319458,
      "learning_rate": 0.00012531242450928133,
      "logits/chosen": -1.0769760608673096,
      "logits/rejected": -1.085452914237976,
      "logps/chosen": -6.943548202514648,
      "logps/rejected": -56.42805480957031,
      "loss": 0.7162372469902039,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4733220636844635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2597489058971405,
      "rewards/margins": 4.188333511352539,
      "rewards/rejected": -3.9285848140716553,
      "step": 524,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8970046452026269,
      "grad_norm": 1.4161630868911743,
      "learning_rate": 0.00012503876086209615,
      "logits/chosen": -1.0991950035095215,
      "logits/rejected": -1.1064820289611816,
      "logps/chosen": -8.520299911499023,
      "logps/rejected": -56.450138092041016,
      "loss": 0.3773810565471649,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2895197868347168,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22713249921798706,
      "rewards/margins": 4.4980998039245605,
      "rewards/rejected": -4.270967483520508,
      "step": 525,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.8987132254792034,
      "grad_norm": 2.5488271713256836,
      "learning_rate": 0.00012476489700564633,
      "logits/chosen": -1.1056780815124512,
      "logits/rejected": -1.1145663261413574,
      "logps/chosen": -7.315359115600586,
      "logps/rejected": -47.98476028442383,
      "loss": 0.3882770240306854,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27460724115371704,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.28682947158813477,
      "rewards/margins": 4.092487812042236,
      "rewards/rejected": -3.8056583404541016,
      "step": 526,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.9004218057557798,
      "grad_norm": 3.121401786804199,
      "learning_rate": 0.00012449083512974,
      "logits/chosen": -1.0872236490249634,
      "logits/rejected": -1.0921869277954102,
      "logps/chosen": -5.041861534118652,
      "logps/rejected": -46.301551818847656,
      "loss": 0.36994799971580505,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2028692662715912,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.258600115776062,
      "rewards/margins": 3.8493764400482178,
      "rewards/rejected": -3.590776205062866,
      "step": 527,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.9021303860323563,
      "grad_norm": 3.0620720386505127,
      "learning_rate": 0.00012421657742576851,
      "logits/chosen": -1.0736360549926758,
      "logits/rejected": -1.0814268589019775,
      "logps/chosen": -11.92237377166748,
      "logps/rejected": -57.635955810546875,
      "loss": 0.4473233222961426,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3409923017024994,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19856709241867065,
      "rewards/margins": 4.668188095092773,
      "rewards/rejected": -4.469620704650879,
      "step": 528,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.9038389663089327,
      "grad_norm": 1.2146646976470947,
      "learning_rate": 0.00012394212608668928,
      "logits/chosen": -1.0623712539672852,
      "logits/rejected": -1.071022391319275,
      "logps/chosen": -7.178973197937012,
      "logps/rejected": -59.53617858886719,
      "loss": 0.25364693999290466,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18632447719573975,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2962036430835724,
      "rewards/margins": 5.080264091491699,
      "rewards/rejected": -4.784060478210449,
      "step": 529,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.9055475465855091,
      "grad_norm": 1.513461709022522,
      "learning_rate": 0.00012366748330700785,
      "logits/chosen": -1.1123794317245483,
      "logits/rejected": -1.1185814142227173,
      "logps/chosen": -8.223485946655273,
      "logps/rejected": -54.725807189941406,
      "loss": 0.411434143781662,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31881242990493774,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.39547258615493774,
      "rewards/margins": 4.645408630371094,
      "rewards/rejected": -4.249935150146484,
      "step": 530,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9072561268620856,
      "grad_norm": 0.8842201232910156,
      "learning_rate": 0.00012339265128276055,
      "logits/chosen": -1.0735728740692139,
      "logits/rejected": -1.0871232748031616,
      "logps/chosen": -4.40582799911499,
      "logps/rejected": -61.02910614013672,
      "loss": 0.2311829924583435,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2082473486661911,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4231488108634949,
      "rewards/margins": 5.340641975402832,
      "rewards/rejected": -4.9174933433532715,
      "step": 531,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.908964707138662,
      "grad_norm": 14.239304542541504,
      "learning_rate": 0.000123117632211497,
      "logits/chosen": -1.108619213104248,
      "logits/rejected": -1.1144882440567017,
      "logps/chosen": -9.097047805786133,
      "logps/rejected": -47.29059600830078,
      "loss": 1.1096312999725342,
      "memory(GiB)": 46.38,
      "nll_loss": 0.6449442505836487,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.08037547022104263,
      "rewards/margins": 3.497230052947998,
      "rewards/rejected": -3.577605724334717,
      "step": 532,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9106732874152385,
      "grad_norm": 4.287862300872803,
      "learning_rate": 0.00012284242829226224,
      "logits/chosen": -1.0771186351776123,
      "logits/rejected": -1.0833611488342285,
      "logps/chosen": -6.26873254776001,
      "logps/rejected": -62.32661437988281,
      "loss": 0.3345887362957001,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2573338449001312,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.36743026971817017,
      "rewards/margins": 5.281978607177734,
      "rewards/rejected": -4.914547920227051,
      "step": 533,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9123818676918148,
      "grad_norm": 12.72948932647705,
      "learning_rate": 0.0001225670417255796,
      "logits/chosen": -1.0811529159545898,
      "logits/rejected": -1.0847760438919067,
      "logps/chosen": -10.871320724487305,
      "logps/rejected": -50.21855926513672,
      "loss": 1.08903169631958,
      "memory(GiB)": 46.38,
      "nll_loss": 0.8011816740036011,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.16750872135162354,
      "rewards/margins": 3.945251941680908,
      "rewards/rejected": -4.112760543823242,
      "step": 534,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9140904479683912,
      "grad_norm": 7.955687522888184,
      "learning_rate": 0.00012229147471343264,
      "logits/chosen": -1.043433666229248,
      "logits/rejected": -1.0497480630874634,
      "logps/chosen": -6.6239705085754395,
      "logps/rejected": -58.87196350097656,
      "loss": 0.3889910876750946,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29023516178131104,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23989100754261017,
      "rewards/margins": 4.792734146118164,
      "rewards/rejected": -4.5528435707092285,
      "step": 535,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9157990282449677,
      "grad_norm": 2.2868025302886963,
      "learning_rate": 0.00012201572945924789,
      "logits/chosen": -1.0578616857528687,
      "logits/rejected": -1.0675410032272339,
      "logps/chosen": -7.148715496063232,
      "logps/rejected": -55.396278381347656,
      "loss": 0.3101382255554199,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24063077569007874,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.27613842487335205,
      "rewards/margins": 4.722238540649414,
      "rewards/rejected": -4.44610071182251,
      "step": 536,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9175076085215441,
      "grad_norm": 1.7533049583435059,
      "learning_rate": 0.00012173980816787704,
      "logits/chosen": -1.0609538555145264,
      "logits/rejected": -1.0723568201065063,
      "logps/chosen": -7.517036437988281,
      "logps/rejected": -55.47805404663086,
      "loss": 0.4190359115600586,
      "memory(GiB)": 46.38,
      "nll_loss": 0.32531875371932983,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.32404839992523193,
      "rewards/margins": 4.4666218757629395,
      "rewards/rejected": -4.142573356628418,
      "step": 537,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9192161887981206,
      "grad_norm": 20.776586532592773,
      "learning_rate": 0.0001214637130455794,
      "logits/chosen": -1.0204284191131592,
      "logits/rejected": -1.029029130935669,
      "logps/chosen": -5.868730545043945,
      "logps/rejected": -53.281490325927734,
      "loss": 0.4754427373409271,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36121079325675964,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.1495516002178192,
      "rewards/margins": 4.174296855926514,
      "rewards/rejected": -4.024745464324951,
      "step": 538,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.920924769074697,
      "grad_norm": 10.966447830200195,
      "learning_rate": 0.00012118744630000419,
      "logits/chosen": -0.9951702356338501,
      "logits/rejected": -1.001932978630066,
      "logps/chosen": -7.8962836265563965,
      "logps/rejected": -54.540828704833984,
      "loss": 0.4527119994163513,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2737759053707123,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.16042691469192505,
      "rewards/margins": 4.194784641265869,
      "rewards/rejected": -4.034358024597168,
      "step": 539,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9226333493512734,
      "grad_norm": 1.1163724660873413,
      "learning_rate": 0.00012091101014017294,
      "logits/chosen": -1.0130131244659424,
      "logits/rejected": -1.022095799446106,
      "logps/chosen": -5.379613399505615,
      "logps/rejected": -63.27645492553711,
      "loss": 0.2223237305879593,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17831042408943176,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31630760431289673,
      "rewards/margins": 4.874075412750244,
      "rewards/rejected": -4.557767868041992,
      "step": 540,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9243419296278499,
      "grad_norm": 1.3874903917312622,
      "learning_rate": 0.00012063440677646185,
      "logits/chosen": -0.9535560607910156,
      "logits/rejected": -0.9632040858268738,
      "logps/chosen": -8.00114917755127,
      "logps/rejected": -55.636810302734375,
      "loss": 0.3135739862918854,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2537074685096741,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2851589322090149,
      "rewards/margins": 4.6183061599731445,
      "rewards/rejected": -4.333147048950195,
      "step": 541,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9260505099044263,
      "grad_norm": 14.624671936035156,
      "learning_rate": 0.00012035763842058403,
      "logits/chosen": -0.9720752835273743,
      "logits/rejected": -0.980399489402771,
      "logps/chosen": -7.485421180725098,
      "logps/rejected": -51.827842712402344,
      "loss": 0.4598942697048187,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3002854585647583,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20470131933689117,
      "rewards/margins": 4.32320499420166,
      "rewards/rejected": -4.118503570556641,
      "step": 542,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9277590901810028,
      "grad_norm": 4.372304439544678,
      "learning_rate": 0.00012008070728557186,
      "logits/chosen": -0.9843500852584839,
      "logits/rejected": -0.991563618183136,
      "logps/chosen": -6.97879695892334,
      "logps/rejected": -51.602195739746094,
      "loss": 0.8653340339660645,
      "memory(GiB)": 46.38,
      "nll_loss": 0.5912885665893555,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.08500701934099197,
      "rewards/margins": 3.892084836959839,
      "rewards/rejected": -3.807077407836914,
      "step": 543,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9294676704575792,
      "grad_norm": 1.468555212020874,
      "learning_rate": 0.00011980361558575935,
      "logits/chosen": -0.9593384265899658,
      "logits/rejected": -0.9710108041763306,
      "logps/chosen": -6.406820297241211,
      "logps/rejected": -64.63932800292969,
      "loss": 0.2843794524669647,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2340080887079239,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2805470824241638,
      "rewards/margins": 5.325027942657471,
      "rewards/rejected": -5.044480800628662,
      "step": 544,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9311762507341556,
      "grad_norm": 2.482309341430664,
      "learning_rate": 0.0001195263655367643,
      "logits/chosen": -0.9956052303314209,
      "logits/rejected": -1.005716323852539,
      "logps/chosen": -5.427254676818848,
      "logps/rejected": -57.17232131958008,
      "loss": 0.29431024193763733,
      "memory(GiB)": 46.38,
      "nll_loss": 0.200710266828537,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.3607883155345917,
      "rewards/margins": 4.842344284057617,
      "rewards/rejected": -4.481555461883545,
      "step": 545,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.932884831010732,
      "grad_norm": 5.6123762130737305,
      "learning_rate": 0.00011924895935547078,
      "logits/chosen": -0.988015353679657,
      "logits/rejected": -0.9978523254394531,
      "logps/chosen": -7.507255554199219,
      "logps/rejected": -57.80442810058594,
      "loss": 0.3716522455215454,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23281218111515045,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2541303336620331,
      "rewards/margins": 4.764814853668213,
      "rewards/rejected": -4.510684490203857,
      "step": 546,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9345934112873084,
      "grad_norm": 5.197413444519043,
      "learning_rate": 0.00011897139926001116,
      "logits/chosen": -1.0102794170379639,
      "logits/rejected": -1.018759846687317,
      "logps/chosen": -4.1510396003723145,
      "logps/rejected": -51.83134841918945,
      "loss": 0.34408482909202576,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2341707944869995,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.30672261118888855,
      "rewards/margins": 4.261148929595947,
      "rewards/rejected": -3.9544262886047363,
      "step": 547,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9363019915638849,
      "grad_norm": 5.588002681732178,
      "learning_rate": 0.0001186936874697486,
      "logits/chosen": -1.0000367164611816,
      "logits/rejected": -1.00547194480896,
      "logps/chosen": -11.38198184967041,
      "logps/rejected": -52.50801086425781,
      "loss": 0.6180362105369568,
      "memory(GiB)": 46.38,
      "nll_loss": 0.49763885140419006,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.26223206520080566,
      "rewards/margins": 4.402937889099121,
      "rewards/rejected": -4.1407060623168945,
      "step": 548,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9380105718404613,
      "grad_norm": 7.8491530418396,
      "learning_rate": 0.00011841582620525917,
      "logits/chosen": -0.9723646640777588,
      "logits/rejected": -0.9810507297515869,
      "logps/chosen": -7.355838775634766,
      "logps/rejected": -52.144691467285156,
      "loss": 0.501491367816925,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3369973599910736,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.2980511486530304,
      "rewards/margins": 4.433564186096191,
      "rewards/rejected": -4.1355133056640625,
      "step": 549,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9397191521170377,
      "grad_norm": 6.467865467071533,
      "learning_rate": 0.00011813781768831411,
      "logits/chosen": -0.9922822117805481,
      "logits/rejected": -1.0177570581436157,
      "logps/chosen": -3.4431815147399902,
      "logps/rejected": -65.48020935058594,
      "loss": 0.18524883687496185,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1471279412508011,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2626737952232361,
      "rewards/margins": 5.496283054351807,
      "rewards/rejected": -5.233609676361084,
      "step": 550,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9414277323936142,
      "grad_norm": 11.377161026000977,
      "learning_rate": 0.00011785966414186216,
      "logits/chosen": -1.0040748119354248,
      "logits/rejected": -1.0146125555038452,
      "logps/chosen": -4.688794136047363,
      "logps/rejected": -56.14875411987305,
      "loss": 0.29910555481910706,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24501614272594452,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1769258677959442,
      "rewards/margins": 4.658725261688232,
      "rewards/rejected": -4.481799125671387,
      "step": 551,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.9431363126701906,
      "grad_norm": 2.027425527572632,
      "learning_rate": 0.00011758136779001166,
      "logits/chosen": -1.0211999416351318,
      "logits/rejected": -1.0322864055633545,
      "logps/chosen": -7.402100563049316,
      "logps/rejected": -57.922523498535156,
      "loss": 0.38571181893348694,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26558855175971985,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.21156170964241028,
      "rewards/margins": 4.732623100280762,
      "rewards/rejected": -4.521061420440674,
      "step": 552,
      "train_speed(iter/s)": 0.005452
    },
    {
      "epoch": 0.9448448929467671,
      "grad_norm": 4.133325576782227,
      "learning_rate": 0.0001173029308580128,
      "logits/chosen": -1.0496547222137451,
      "logits/rejected": -1.0650711059570312,
      "logps/chosen": -7.616898536682129,
      "logps/rejected": -58.14318084716797,
      "loss": 0.4242968261241913,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3174532949924469,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.20163990557193756,
      "rewards/margins": 4.849511623382568,
      "rewards/rejected": -4.647871971130371,
      "step": 553,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9465534732233435,
      "grad_norm": 1.6775513887405396,
      "learning_rate": 0.00011702435557223987,
      "logits/chosen": -1.0214565992355347,
      "logits/rejected": -1.0286892652511597,
      "logps/chosen": -5.073369979858398,
      "logps/rejected": -54.65155792236328,
      "loss": 0.2832258641719818,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19457288086414337,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.30036842823028564,
      "rewards/margins": 4.594720840454102,
      "rewards/rejected": -4.2943525314331055,
      "step": 554,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9482620534999199,
      "grad_norm": 1.8203599452972412,
      "learning_rate": 0.00011674564416017344,
      "logits/chosen": -0.9795448184013367,
      "logits/rejected": -0.9978064298629761,
      "logps/chosen": -6.611621379852295,
      "logps/rejected": -63.19961929321289,
      "loss": 0.28948503732681274,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23222242295742035,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.33579951524734497,
      "rewards/margins": 5.128011703491211,
      "rewards/rejected": -4.792212009429932,
      "step": 555,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9499706337764964,
      "grad_norm": 4.286935329437256,
      "learning_rate": 0.00011646679885038252,
      "logits/chosen": -0.9947451949119568,
      "logits/rejected": -1.0057193040847778,
      "logps/chosen": -6.576064586639404,
      "logps/rejected": -48.98035430908203,
      "loss": 0.6356810927391052,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4307587146759033,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.24798831343650818,
      "rewards/margins": 3.9605040550231934,
      "rewards/rejected": -3.7125158309936523,
      "step": 556,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9516792140530728,
      "grad_norm": 26.787351608276367,
      "learning_rate": 0.00011618782187250681,
      "logits/chosen": -0.9705215692520142,
      "logits/rejected": -0.9764195680618286,
      "logps/chosen": -10.289443016052246,
      "logps/rejected": -46.06808853149414,
      "loss": 0.7001438736915588,
      "memory(GiB)": 46.38,
      "nll_loss": 0.47328078746795654,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.12188911437988281,
      "rewards/margins": 3.825579881668091,
      "rewards/rejected": -3.703691005706787,
      "step": 557,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9533877943296492,
      "grad_norm": 1.5328869819641113,
      "learning_rate": 0.00011590871545723876,
      "logits/chosen": -0.970535159111023,
      "logits/rejected": -0.9791247248649597,
      "logps/chosen": -5.036118984222412,
      "logps/rejected": -53.236968994140625,
      "loss": 0.3036826252937317,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19022338092327118,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.39716899394989014,
      "rewards/margins": 4.327749729156494,
      "rewards/rejected": -3.9305806159973145,
      "step": 558,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9550963746062257,
      "grad_norm": 1.2964873313903809,
      "learning_rate": 0.00011562948183630581,
      "logits/chosen": -0.9282693862915039,
      "logits/rejected": -0.9432880878448486,
      "logps/chosen": -3.670807361602783,
      "logps/rejected": -60.272396087646484,
      "loss": 0.17175671458244324,
      "memory(GiB)": 46.38,
      "nll_loss": 0.13948416709899902,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2401636391878128,
      "rewards/margins": 4.895763397216797,
      "rewards/rejected": -4.655599594116211,
      "step": 559,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.956804954882802,
      "grad_norm": 1.8788788318634033,
      "learning_rate": 0.0001153501232424526,
      "logits/chosen": -0.9622473120689392,
      "logits/rejected": -0.9669633507728577,
      "logps/chosen": -8.018237113952637,
      "logps/rejected": -55.00972366333008,
      "loss": 0.45394715666770935,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3820911645889282,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3880501389503479,
      "rewards/margins": 4.521093845367432,
      "rewards/rejected": -4.133044242858887,
      "step": 560,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9585135351593785,
      "grad_norm": 14.007570266723633,
      "learning_rate": 0.000115070641909423,
      "logits/chosen": -0.9567843079566956,
      "logits/rejected": -0.9724023938179016,
      "logps/chosen": -6.963770866394043,
      "logps/rejected": -59.60305404663086,
      "loss": 0.397840678691864,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3546963334083557,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.20955303311347961,
      "rewards/margins": 4.792242050170898,
      "rewards/rejected": -4.582688808441162,
      "step": 561,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9602221154359549,
      "grad_norm": 7.802319526672363,
      "learning_rate": 0.0001147910400719423,
      "logits/chosen": -0.9432657957077026,
      "logits/rejected": -0.9551844596862793,
      "logps/chosen": -6.495790004730225,
      "logps/rejected": -60.8466682434082,
      "loss": 0.46896928548812866,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4095912277698517,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.1675097942352295,
      "rewards/margins": 4.8893537521362305,
      "rewards/rejected": -4.721844673156738,
      "step": 562,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9619306957125313,
      "grad_norm": 2.0097382068634033,
      "learning_rate": 0.00011451131996569928,
      "logits/chosen": -0.9693796634674072,
      "logits/rejected": -0.9789756536483765,
      "logps/chosen": -11.421247482299805,
      "logps/rejected": -59.551841735839844,
      "loss": 0.4494234621524811,
      "memory(GiB)": 46.38,
      "nll_loss": 0.36230331659317017,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.04363319277763367,
      "rewards/margins": 4.113037586212158,
      "rewards/rejected": -4.069404602050781,
      "step": 563,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9636392759891078,
      "grad_norm": 6.29232120513916,
      "learning_rate": 0.00011423148382732853,
      "logits/chosen": -0.9759683012962341,
      "logits/rejected": -0.9832198023796082,
      "logps/chosen": -8.147719383239746,
      "logps/rejected": -52.644805908203125,
      "loss": 0.5432403683662415,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4694908857345581,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.07839629054069519,
      "rewards/margins": 4.158172607421875,
      "rewards/rejected": -4.079776763916016,
      "step": 564,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9653478562656842,
      "grad_norm": 2.7234086990356445,
      "learning_rate": 0.00011395153389439233,
      "logits/chosen": -0.970282256603241,
      "logits/rejected": -0.9783824682235718,
      "logps/chosen": -5.9536566734313965,
      "logps/rejected": -52.36622619628906,
      "loss": 0.3171547055244446,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24613326787948608,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.30720555782318115,
      "rewards/margins": 4.494471549987793,
      "rewards/rejected": -4.1872663497924805,
      "step": 565,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9670564365422607,
      "grad_norm": 15.189948081970215,
      "learning_rate": 0.00011367147240536285,
      "logits/chosen": -0.9521063566207886,
      "logits/rejected": -0.9623022079467773,
      "logps/chosen": -5.562067031860352,
      "logps/rejected": -60.893978118896484,
      "loss": 0.4072427451610565,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34856539964675903,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3333992063999176,
      "rewards/margins": 5.3980231285095215,
      "rewards/rejected": -5.064624309539795,
      "step": 566,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9687650168188371,
      "grad_norm": 4.7368316650390625,
      "learning_rate": 0.00011339130159960434,
      "logits/chosen": -1.0292445421218872,
      "logits/rejected": -1.0421253442764282,
      "logps/chosen": -5.4102301597595215,
      "logps/rejected": -62.65816116333008,
      "loss": 0.5825440883636475,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4267558455467224,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2507926821708679,
      "rewards/margins": 5.191431522369385,
      "rewards/rejected": -4.940639495849609,
      "step": 567,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9704735970954135,
      "grad_norm": 1.5294198989868164,
      "learning_rate": 0.00011311102371735509,
      "logits/chosen": -0.960658073425293,
      "logits/rejected": -0.9676105976104736,
      "logps/chosen": -9.309149742126465,
      "logps/rejected": -56.24421691894531,
      "loss": 0.3837515413761139,
      "memory(GiB)": 46.38,
      "nll_loss": 0.30812618136405945,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.09037159383296967,
      "rewards/margins": 4.825429439544678,
      "rewards/rejected": -4.735057830810547,
      "step": 568,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.97218217737199,
      "grad_norm": 2.553351640701294,
      "learning_rate": 0.00011283064099970959,
      "logits/chosen": -0.9884815216064453,
      "logits/rejected": -0.9964404702186584,
      "logps/chosen": -15.506885528564453,
      "logps/rejected": -61.829627990722656,
      "loss": 0.5028576254844666,
      "memory(GiB)": 46.38,
      "nll_loss": 0.430748850107193,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.25180143117904663,
      "rewards/margins": 5.249931812286377,
      "rewards/rejected": -4.998130798339844,
      "step": 569,
      "train_speed(iter/s)": 0.005453
    },
    {
      "epoch": 0.9738907576485664,
      "grad_norm": 10.381235122680664,
      "learning_rate": 0.00011255015568860052,
      "logits/chosen": -1.0335317850112915,
      "logits/rejected": -1.04970383644104,
      "logps/chosen": -3.703486442565918,
      "logps/rejected": -68.1993637084961,
      "loss": 0.29912862181663513,
      "memory(GiB)": 46.38,
      "nll_loss": 0.21877992153167725,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.30584433674812317,
      "rewards/margins": 5.8483123779296875,
      "rewards/rejected": -5.542468547821045,
      "step": 570,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9755993379251429,
      "grad_norm": 1.9842922687530518,
      "learning_rate": 0.00011226957002678102,
      "logits/chosen": -1.0329009294509888,
      "logits/rejected": -1.0508722066879272,
      "logps/chosen": -4.218231678009033,
      "logps/rejected": -74.08231353759766,
      "loss": 0.2076101154088974,
      "memory(GiB)": 46.38,
      "nll_loss": 0.15104490518569946,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.24029362201690674,
      "rewards/margins": 6.227653980255127,
      "rewards/rejected": -5.987360954284668,
      "step": 571,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9773079182017193,
      "grad_norm": 0.844889760017395,
      "learning_rate": 0.00011198888625780651,
      "logits/chosen": -1.0135738849639893,
      "logits/rejected": -1.0236599445343018,
      "logps/chosen": -6.657075881958008,
      "logps/rejected": -70.59235382080078,
      "loss": 0.21408958733081818,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19797447323799133,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3546680510044098,
      "rewards/margins": 6.002138137817383,
      "rewards/rejected": -5.647469520568848,
      "step": 572,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9790164984782956,
      "grad_norm": 20.870840072631836,
      "learning_rate": 0.00011170810662601696,
      "logits/chosen": -0.9645272493362427,
      "logits/rejected": -0.9769738912582397,
      "logps/chosen": -6.054335594177246,
      "logps/rejected": -59.02510070800781,
      "loss": 0.3599255681037903,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2686828076839447,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2436637580394745,
      "rewards/margins": 5.1186089515686035,
      "rewards/rejected": -4.874945163726807,
      "step": 573,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9807250787548721,
      "grad_norm": 10.852333068847656,
      "learning_rate": 0.00011142723337651881,
      "logits/chosen": -1.0215522050857544,
      "logits/rejected": -1.031611680984497,
      "logps/chosen": -7.273950576782227,
      "logps/rejected": -61.101802825927734,
      "loss": 0.5884179472923279,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3813873827457428,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.14103633165359497,
      "rewards/margins": 5.116168022155762,
      "rewards/rejected": -4.975131988525391,
      "step": 574,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9824336590314485,
      "grad_norm": 2.253772258758545,
      "learning_rate": 0.00011114626875516711,
      "logits/chosen": -0.9972852468490601,
      "logits/rejected": -1.0096957683563232,
      "logps/chosen": -7.324893474578857,
      "logps/rejected": -66.20896911621094,
      "loss": 0.37433934211730957,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3080398738384247,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.436337947845459,
      "rewards/margins": 5.712600231170654,
      "rewards/rejected": -5.276262283325195,
      "step": 575,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.984142239308025,
      "grad_norm": 8.405837059020996,
      "learning_rate": 0.00011086521500854745,
      "logits/chosen": -0.9975009560585022,
      "logits/rejected": -1.0023764371871948,
      "logps/chosen": -8.61086654663086,
      "logps/rejected": -56.08827209472656,
      "loss": 0.6244361400604248,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4514012336730957,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19428743422031403,
      "rewards/margins": 4.60284948348999,
      "rewards/rejected": -4.408562183380127,
      "step": 576,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9858508195846014,
      "grad_norm": 1.896874189376831,
      "learning_rate": 0.00011058407438395812,
      "logits/chosen": -1.0023829936981201,
      "logits/rejected": -1.0130419731140137,
      "logps/chosen": -8.414669036865234,
      "logps/rejected": -58.671142578125,
      "loss": 0.3457658290863037,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25555044412612915,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.2798009514808655,
      "rewards/margins": 4.668698310852051,
      "rewards/rejected": -4.388896942138672,
      "step": 577,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9875593998611778,
      "grad_norm": 4.986742973327637,
      "learning_rate": 0.00011030284912939206,
      "logits/chosen": -0.9768277406692505,
      "logits/rejected": -0.9915848970413208,
      "logps/chosen": -4.5320329666137695,
      "logps/rejected": -63.7336540222168,
      "loss": 0.2967624068260193,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18453125655651093,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21622657775878906,
      "rewards/margins": 5.409724235534668,
      "rewards/rejected": -5.193497657775879,
      "step": 578,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9892679801377543,
      "grad_norm": 5.255149841308594,
      "learning_rate": 0.00011002154149351888,
      "logits/chosen": -0.9702717661857605,
      "logits/rejected": -0.9778293371200562,
      "logps/chosen": -5.819952964782715,
      "logps/rejected": -53.747703552246094,
      "loss": 0.603337824344635,
      "memory(GiB)": 46.38,
      "nll_loss": 0.4150553345680237,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.15679959952831268,
      "rewards/margins": 4.582903861999512,
      "rewards/rejected": -4.426104545593262,
      "step": 579,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9909765604143307,
      "grad_norm": 0.9375675916671753,
      "learning_rate": 0.00010974015372566698,
      "logits/chosen": -0.9859532713890076,
      "logits/rejected": -0.9950516223907471,
      "logps/chosen": -5.118361473083496,
      "logps/rejected": -57.94718551635742,
      "loss": 0.24254225194454193,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20820516347885132,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31492820382118225,
      "rewards/margins": 5.071423053741455,
      "rewards/rejected": -4.756494522094727,
      "step": 580,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9926851406909072,
      "grad_norm": 1.5512089729309082,
      "learning_rate": 0.00010945868807580542,
      "logits/chosen": -0.9601040482521057,
      "logits/rejected": -0.9705081582069397,
      "logps/chosen": -5.833887100219727,
      "logps/rejected": -60.05228805541992,
      "loss": 0.2632489502429962,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23256337642669678,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.16433978080749512,
      "rewards/margins": 4.929656028747559,
      "rewards/rejected": -4.765316009521484,
      "step": 581,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9943937209674836,
      "grad_norm": 1.419494867324829,
      "learning_rate": 0.00010917714679452608,
      "logits/chosen": -0.9395344853401184,
      "logits/rejected": -0.9534673094749451,
      "logps/chosen": -2.8693313598632812,
      "logps/rejected": -65.45692443847656,
      "loss": 0.17592380940914154,
      "memory(GiB)": 46.38,
      "nll_loss": 0.12681818008422852,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.23711417615413666,
      "rewards/margins": 5.223657608032227,
      "rewards/rejected": -4.986542701721191,
      "step": 582,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.99610230124406,
      "grad_norm": 1.7051347494125366,
      "learning_rate": 0.0001088955321330255,
      "logits/chosen": -0.9301373362541199,
      "logits/rejected": -0.936238706111908,
      "logps/chosen": -8.611891746520996,
      "logps/rejected": -52.640625,
      "loss": 0.3892059028148651,
      "memory(GiB)": 46.38,
      "nll_loss": 0.289109468460083,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.19621115922927856,
      "rewards/margins": 4.376561164855957,
      "rewards/rejected": -4.180349826812744,
      "step": 583,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9978108815206365,
      "grad_norm": 1.5059183835983276,
      "learning_rate": 0.00010861384634308704,
      "logits/chosen": -0.9690762758255005,
      "logits/rejected": -0.9810917377471924,
      "logps/chosen": -5.192257404327393,
      "logps/rejected": -54.9718017578125,
      "loss": 0.31561902165412903,
      "memory(GiB)": 46.38,
      "nll_loss": 0.22976836562156677,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3075409531593323,
      "rewards/margins": 4.586206436157227,
      "rewards/rejected": -4.278665065765381,
      "step": 584,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 0.9995194617972128,
      "grad_norm": 2.554860830307007,
      "learning_rate": 0.00010833209167706276,
      "logits/chosen": -0.9669089317321777,
      "logits/rejected": -0.9702744483947754,
      "logps/chosen": -6.821906089782715,
      "logps/rejected": -59.51226043701172,
      "loss": 0.2586469352245331,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2351158708333969,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.24566525220870972,
      "rewards/margins": 5.20125675201416,
      "rewards/rejected": -4.955592155456543,
      "step": 585,
      "train_speed(iter/s)": 0.005454
    },
    {
      "epoch": 1.0,
      "grad_norm": 2.554860830307007,
      "learning_rate": 0.00010805027038785544,
      "logits/chosen": -0.9275478720664978,
      "logits/rejected": -0.9432591199874878,
      "logps/chosen": -2.7973599433898926,
      "logps/rejected": -61.20429229736328,
      "loss": 0.04466842859983444,
      "memory(GiB)": 46.38,
      "nll_loss": 0.10808717459440231,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.17456038296222687,
      "rewards/margins": 5.222509384155273,
      "rewards/rejected": -5.047948837280273,
      "step": 586,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0017085802765764,
      "grad_norm": 1.5154718160629272,
      "learning_rate": 0.00010776838472890065,
      "logits/chosen": -0.9898529052734375,
      "logits/rejected": -0.9935244917869568,
      "logps/chosen": -9.510120391845703,
      "logps/rejected": -54.0595588684082,
      "loss": 0.3424808084964752,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29532164335250854,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.611236035823822,
      "rewards/margins": 4.841841697692871,
      "rewards/rejected": -4.2306060791015625,
      "step": 587,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0034171605531528,
      "grad_norm": 0.7335729598999023,
      "learning_rate": 0.00010748643695414854,
      "logits/chosen": -0.9169901609420776,
      "logits/rejected": -0.9291804432868958,
      "logps/chosen": -2.1846466064453125,
      "logps/rejected": -68.7997817993164,
      "loss": 0.15192590653896332,
      "memory(GiB)": 46.38,
      "nll_loss": 0.14130420982837677,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.25540679693222046,
      "rewards/margins": 5.668377876281738,
      "rewards/rejected": -5.4129719734191895,
      "step": 588,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0051257408297294,
      "grad_norm": 1.179013967514038,
      "learning_rate": 0.00010720442931804604,
      "logits/chosen": -0.9883674383163452,
      "logits/rejected": -0.9982539415359497,
      "logps/chosen": -5.284873008728027,
      "logps/rejected": -61.78940200805664,
      "loss": 0.2318183332681656,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19733554124832153,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5069313049316406,
      "rewards/margins": 5.4225850105285645,
      "rewards/rejected": -4.915654182434082,
      "step": 589,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0068343211063058,
      "grad_norm": 0.8819759488105774,
      "learning_rate": 0.00010692236407551868,
      "logits/chosen": -1.0039994716644287,
      "logits/rejected": -1.010361909866333,
      "logps/chosen": -3.940282106399536,
      "logps/rejected": -59.902339935302734,
      "loss": 0.18675497174263,
      "memory(GiB)": 46.38,
      "nll_loss": 0.14847590029239655,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6330046057701111,
      "rewards/margins": 5.498729228973389,
      "rewards/rejected": -4.865724086761475,
      "step": 590,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0085429013828822,
      "grad_norm": 14.215339660644531,
      "learning_rate": 0.00010664024348195267,
      "logits/chosen": -0.9927942752838135,
      "logits/rejected": -1.0032246112823486,
      "logps/chosen": -5.610966682434082,
      "logps/rejected": -61.65888977050781,
      "loss": 0.3902447521686554,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2995155453681946,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4226346015930176,
      "rewards/margins": 5.410132884979248,
      "rewards/rejected": -4.9874982833862305,
      "step": 591,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0102514816594586,
      "grad_norm": 2.305922031402588,
      "learning_rate": 0.0001063580697931767,
      "logits/chosen": -1.07249915599823,
      "logits/rejected": -1.077643632888794,
      "logps/chosen": -5.720154762268066,
      "logps/rejected": -66.6709976196289,
      "loss": 0.2512580454349518,
      "memory(GiB)": 46.38,
      "nll_loss": 0.22107577323913574,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31614840030670166,
      "rewards/margins": 6.049639701843262,
      "rewards/rejected": -5.73349142074585,
      "step": 592,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.011960061936035,
      "grad_norm": 1.6738953590393066,
      "learning_rate": 0.00010607584526544412,
      "logits/chosen": -1.0542024374008179,
      "logits/rejected": -1.066709041595459,
      "logps/chosen": -6.184117317199707,
      "logps/rejected": -70.54666137695312,
      "loss": 0.28743675351142883,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23315902054309845,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.4204199016094208,
      "rewards/margins": 6.133965015411377,
      "rewards/rejected": -5.713545799255371,
      "step": 593,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0136686422126115,
      "grad_norm": 11.543700218200684,
      "learning_rate": 0.00010579357215541468,
      "logits/chosen": -1.1367042064666748,
      "logits/rejected": -1.1463769674301147,
      "logps/chosen": -9.397639274597168,
      "logps/rejected": -76.76417541503906,
      "loss": 0.3570110499858856,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2863011360168457,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.7818154096603394,
      "rewards/margins": 6.958240032196045,
      "rewards/rejected": -6.176424503326416,
      "step": 594,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.015377222489188,
      "grad_norm": 1.009209394454956,
      "learning_rate": 0.00010551125272013665,
      "logits/chosen": -1.153163194656372,
      "logits/rejected": -1.1620734930038452,
      "logps/chosen": -5.096438884735107,
      "logps/rejected": -71.95016479492188,
      "loss": 0.20938211679458618,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17731687426567078,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2625049948692322,
      "rewards/margins": 6.220973491668701,
      "rewards/rejected": -5.958468437194824,
      "step": 595,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0170858027657643,
      "grad_norm": 0.7603577375411987,
      "learning_rate": 0.00010522888921702873,
      "logits/chosen": -1.175655484199524,
      "logits/rejected": -1.1842759847640991,
      "logps/chosen": -6.609786033630371,
      "logps/rejected": -75.14688873291016,
      "loss": 0.2439320832490921,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23162446916103363,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5667248964309692,
      "rewards/margins": 7.023657321929932,
      "rewards/rejected": -6.456932544708252,
      "step": 596,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0187943830423407,
      "grad_norm": 1.0994817018508911,
      "learning_rate": 0.00010494648390386199,
      "logits/chosen": -1.1729586124420166,
      "logits/rejected": -1.1890482902526855,
      "logps/chosen": -6.585988521575928,
      "logps/rejected": -81.09597778320312,
      "loss": 0.27688947319984436,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24933534860610962,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.428009033203125,
      "rewards/margins": 6.950448513031006,
      "rewards/rejected": -6.522439002990723,
      "step": 597,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.020502963318917,
      "grad_norm": 0.9391170740127563,
      "learning_rate": 0.00010466403903874176,
      "logits/chosen": -1.1939568519592285,
      "logits/rejected": -1.2136722803115845,
      "logps/chosen": -3.855241298675537,
      "logps/rejected": -85.42668151855469,
      "loss": 0.17003044486045837,
      "memory(GiB)": 46.38,
      "nll_loss": 0.15238091349601746,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41148266196250916,
      "rewards/margins": 7.366979598999023,
      "rewards/rejected": -6.955496788024902,
      "step": 598,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.0222115435954937,
      "grad_norm": 4.469460964202881,
      "learning_rate": 0.00010438155688008966,
      "logits/chosen": -1.2343566417694092,
      "logits/rejected": -1.2449347972869873,
      "logps/chosen": -7.440985202789307,
      "logps/rejected": -82.56564331054688,
      "loss": 0.28530043363571167,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27977192401885986,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.38241153955459595,
      "rewards/margins": 7.438549995422363,
      "rewards/rejected": -7.056138515472412,
      "step": 599,
      "train_speed(iter/s)": 0.005461
    },
    {
      "epoch": 1.02392012387207,
      "grad_norm": 5.903979778289795,
      "learning_rate": 0.00010409903968662551,
      "logits/chosen": -1.2281779050827026,
      "logits/rejected": -1.2403244972229004,
      "logps/chosen": -6.364772796630859,
      "logps/rejected": -71.89556884765625,
      "loss": 0.30793577432632446,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2553955316543579,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6146920919418335,
      "rewards/margins": 6.622899532318115,
      "rewards/rejected": -6.00820779800415,
      "step": 600,
      "train_speed(iter/s)": 0.005462
    },
    {
      "epoch": 1.02392012387207,
      "eval_logits/chosen": -1.2626844644546509,
      "eval_logits/rejected": -1.2754260301589966,
      "eval_logps/chosen": -6.6583075523376465,
      "eval_logps/rejected": -78.41804504394531,
      "eval_loss": 0.3543238043785095,
      "eval_nll_loss": 0.3127117455005646,
      "eval_rewards/accuracies": 0.9894179701805115,
      "eval_rewards/chosen": 0.20366309583187103,
      "eval_rewards/margins": 6.702877044677734,
      "eval_rewards/rejected": -6.4992146492004395,
      "eval_runtime": 442.0867,
      "eval_samples_per_second": 0.428,
      "eval_steps_per_second": 0.428,
      "step": 600
    },
    {
      "epoch": 1.0256287041486465,
      "grad_norm": 3.7482240200042725,
      "learning_rate": 0.00010381648971734923,
      "logits/chosen": -1.2164467573165894,
      "logits/rejected": -1.234466552734375,
      "logps/chosen": -3.149059534072876,
      "logps/rejected": -82.25542449951172,
      "loss": 0.46678465604782104,
      "memory(GiB)": 46.38,
      "nll_loss": 0.451040655374527,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.317766934633255,
      "rewards/margins": 7.278980255126953,
      "rewards/rejected": -6.961214065551758,
      "step": 601,
      "train_speed(iter/s)": 0.005439
    },
    {
      "epoch": 1.0273372844252229,
      "grad_norm": 3.6884841918945312,
      "learning_rate": 0.0001035339092315229,
      "logits/chosen": -1.278205156326294,
      "logits/rejected": -1.290305733680725,
      "logps/chosen": -3.5693697929382324,
      "logps/rejected": -86.19032287597656,
      "loss": 0.13512854278087616,
      "memory(GiB)": 46.38,
      "nll_loss": 0.13316480815410614,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.22110113501548767,
      "rewards/margins": 7.681395530700684,
      "rewards/rejected": -7.460294723510742,
      "step": 602,
      "train_speed(iter/s)": 0.005439
    },
    {
      "epoch": 1.0290458647017993,
      "grad_norm": 9.637324333190918,
      "learning_rate": 0.0001032513004886525,
      "logits/chosen": -1.28762686252594,
      "logits/rejected": -1.2962695360183716,
      "logps/chosen": -4.65502405166626,
      "logps/rejected": -75.79096984863281,
      "loss": 0.48636460304260254,
      "memory(GiB)": 46.38,
      "nll_loss": 0.350303053855896,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.45867210626602173,
      "rewards/margins": 6.852001190185547,
      "rewards/rejected": -6.393328666687012,
      "step": 603,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0307544449783759,
      "grad_norm": 0.6257339119911194,
      "learning_rate": 0.00010296866574847005,
      "logits/chosen": -1.2499245405197144,
      "logits/rejected": -1.2687467336654663,
      "logps/chosen": -5.072188854217529,
      "logps/rejected": -90.28076171875,
      "loss": 0.17035354673862457,
      "memory(GiB)": 46.38,
      "nll_loss": 0.16570714116096497,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.48079997301101685,
      "rewards/margins": 7.802813529968262,
      "rewards/rejected": -7.3220133781433105,
      "step": 604,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0324630252549523,
      "grad_norm": 0.940476655960083,
      "learning_rate": 0.00010268600727091537,
      "logits/chosen": -1.2344437837600708,
      "logits/rejected": -1.2520300149917603,
      "logps/chosen": -4.035116195678711,
      "logps/rejected": -86.37598419189453,
      "loss": 0.3515896201133728,
      "memory(GiB)": 46.38,
      "nll_loss": 0.253312885761261,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.04786258935928345,
      "rewards/margins": 7.340476036071777,
      "rewards/rejected": -7.292613506317139,
      "step": 605,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0341716055315286,
      "grad_norm": 27.096158981323242,
      "learning_rate": 0.00010240332731611813,
      "logits/chosen": -1.2474778890609741,
      "logits/rejected": -1.2702710628509521,
      "logps/chosen": -2.4200680255889893,
      "logps/rejected": -89.45378875732422,
      "loss": 0.21904058754444122,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1694769710302353,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3186418414115906,
      "rewards/margins": 8.092009544372559,
      "rewards/rejected": -7.773367881774902,
      "step": 606,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.035880185808105,
      "grad_norm": 12.026522636413574,
      "learning_rate": 0.00010212062814437971,
      "logits/chosen": -1.2728394269943237,
      "logits/rejected": -1.2822685241699219,
      "logps/chosen": -7.859330177307129,
      "logps/rejected": -74.43230438232422,
      "loss": 0.33141881227493286,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2813712954521179,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2280624508857727,
      "rewards/margins": 6.465841293334961,
      "rewards/rejected": -6.237778663635254,
      "step": 607,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0375887660846814,
      "grad_norm": 0.9331662654876709,
      "learning_rate": 0.00010183791201615516,
      "logits/chosen": -1.2525206804275513,
      "logits/rejected": -1.2622168064117432,
      "logps/chosen": -5.9406023025512695,
      "logps/rejected": -74.23300170898438,
      "loss": 0.3456898033618927,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3051641583442688,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2724432051181793,
      "rewards/margins": 6.436144828796387,
      "rewards/rejected": -6.16370153427124,
      "step": 608,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.039297346361258,
      "grad_norm": 7.533527851104736,
      "learning_rate": 0.0001015551811920351,
      "logits/chosen": -1.2520793676376343,
      "logits/rejected": -1.2674461603164673,
      "logps/chosen": -5.58453369140625,
      "logps/rejected": -76.28759765625,
      "loss": 0.2248965948820114,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19207513332366943,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4491349458694458,
      "rewards/margins": 6.884250640869141,
      "rewards/rejected": -6.435115814208984,
      "step": 609,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0410059266378344,
      "grad_norm": 1.3162251710891724,
      "learning_rate": 0.00010127243793272767,
      "logits/chosen": -1.2570934295654297,
      "logits/rejected": -1.279131531715393,
      "logps/chosen": -4.297021865844727,
      "logps/rejected": -101.56732177734375,
      "loss": 0.19886673986911774,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18678924441337585,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.42774003744125366,
      "rewards/margins": 9.003313064575195,
      "rewards/rejected": -8.575572967529297,
      "step": 610,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0427145069144108,
      "grad_norm": 0.7907546162605286,
      "learning_rate": 0.00010098968449904042,
      "logits/chosen": -1.2609025239944458,
      "logits/rejected": -1.277443528175354,
      "logps/chosen": -3.8169775009155273,
      "logps/rejected": -84.62374877929688,
      "loss": 0.1453225314617157,
      "memory(GiB)": 46.38,
      "nll_loss": 0.13972803950309753,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.31030404567718506,
      "rewards/margins": 7.288273811340332,
      "rewards/rejected": -6.977969169616699,
      "step": 611,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0444230871909872,
      "grad_norm": 0.6729239821434021,
      "learning_rate": 0.00010070692315186232,
      "logits/chosen": -1.258347511291504,
      "logits/rejected": -1.2737245559692383,
      "logps/chosen": -4.114006042480469,
      "logps/rejected": -84.53619384765625,
      "loss": 0.20619376003742218,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1795843541622162,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.29713669419288635,
      "rewards/margins": 7.501402378082275,
      "rewards/rejected": -7.20426607131958,
      "step": 612,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0461316674675636,
      "grad_norm": 6.327593803405762,
      "learning_rate": 0.00010042415615214553,
      "logits/chosen": -1.2444207668304443,
      "logits/rejected": -1.2557777166366577,
      "logps/chosen": -4.246981620788574,
      "logps/rejected": -79.84947204589844,
      "loss": 0.2601467967033386,
      "memory(GiB)": 46.38,
      "nll_loss": 0.23486030101776123,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4159022867679596,
      "rewards/margins": 7.286006927490234,
      "rewards/rejected": -6.8701043128967285,
      "step": 613,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.0478402477441402,
      "grad_norm": 2.7996058464050293,
      "learning_rate": 0.00010014138576088746,
      "logits/chosen": -1.2451854944229126,
      "logits/rejected": -1.2592599391937256,
      "logps/chosen": -2.3617138862609863,
      "logps/rejected": -78.9321517944336,
      "loss": 0.1446584165096283,
      "memory(GiB)": 46.38,
      "nll_loss": 0.11354342848062515,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.33388298749923706,
      "rewards/margins": 7.155437469482422,
      "rewards/rejected": -6.821554183959961,
      "step": 614,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0495488280207166,
      "grad_norm": 53.42617416381836,
      "learning_rate": 9.98586142391126e-05,
      "logits/chosen": -1.2627508640289307,
      "logits/rejected": -1.282650351524353,
      "logps/chosen": -3.4703731536865234,
      "logps/rejected": -82.5564193725586,
      "loss": 0.3693418502807617,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2534872591495514,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.1645037829875946,
      "rewards/margins": 6.967869758605957,
      "rewards/rejected": -6.803365707397461,
      "step": 615,
      "train_speed(iter/s)": 0.00544
    },
    {
      "epoch": 1.051257408297293,
      "grad_norm": 4.493410110473633,
      "learning_rate": 9.95758438478545e-05,
      "logits/chosen": -1.2520884275436401,
      "logits/rejected": -1.2608726024627686,
      "logps/chosen": -5.0455756187438965,
      "logps/rejected": -74.4554443359375,
      "loss": 0.18476612865924835,
      "memory(GiB)": 46.38,
      "nll_loss": 0.17582294344902039,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3595569133758545,
      "rewards/margins": 6.679653644561768,
      "rewards/rejected": -6.320096969604492,
      "step": 616,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0529659885738694,
      "grad_norm": 1.5083187818527222,
      "learning_rate": 9.929307684813769e-05,
      "logits/chosen": -1.3061866760253906,
      "logits/rejected": -1.3120005130767822,
      "logps/chosen": -5.171699047088623,
      "logps/rejected": -67.46263885498047,
      "loss": 0.2588375508785248,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20567472279071808,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5596597194671631,
      "rewards/margins": 6.103704452514648,
      "rewards/rejected": -5.544044494628906,
      "step": 617,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0546745688504457,
      "grad_norm": 16.47061538696289,
      "learning_rate": 9.901031550095958e-05,
      "logits/chosen": -1.252734899520874,
      "logits/rejected": -1.266066551208496,
      "logps/chosen": -6.780295372009277,
      "logps/rejected": -73.7783203125,
      "loss": 0.47020223736763,
      "memory(GiB)": 46.38,
      "nll_loss": 0.3376345634460449,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.21828973293304443,
      "rewards/margins": 6.299534797668457,
      "rewards/rejected": -6.081245422363281,
      "step": 618,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0563831491270224,
      "grad_norm": 1.1983541250228882,
      "learning_rate": 9.872756206727236e-05,
      "logits/chosen": -1.313442349433899,
      "logits/rejected": -1.321502447128296,
      "logps/chosen": -5.958905220031738,
      "logps/rejected": -76.18379211425781,
      "loss": 0.20255246758460999,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18408368527889252,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2792333960533142,
      "rewards/margins": 6.787381172180176,
      "rewards/rejected": -6.508147716522217,
      "step": 619,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0580917294035987,
      "grad_norm": 1.8655586242675781,
      "learning_rate": 9.844481880796491e-05,
      "logits/chosen": -1.2791587114334106,
      "logits/rejected": -1.285279393196106,
      "logps/chosen": -3.4074649810791016,
      "logps/rejected": -68.55343627929688,
      "loss": 0.19738541543483734,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1737406849861145,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3685428500175476,
      "rewards/margins": 6.238713264465332,
      "rewards/rejected": -5.870171070098877,
      "step": 620,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0598003096801751,
      "grad_norm": 11.921148300170898,
      "learning_rate": 9.816208798384485e-05,
      "logits/chosen": -1.301615595817566,
      "logits/rejected": -1.3090208768844604,
      "logps/chosen": -8.143779754638672,
      "logps/rejected": -69.29180145263672,
      "loss": 0.321319043636322,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29410284757614136,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3241105079650879,
      "rewards/margins": 5.988301753997803,
      "rewards/rejected": -5.664191246032715,
      "step": 621,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0615088899567515,
      "grad_norm": 3.473475456237793,
      "learning_rate": 9.787937185562033e-05,
      "logits/chosen": -1.292022705078125,
      "logits/rejected": -1.3112902641296387,
      "logps/chosen": -2.760141372680664,
      "logps/rejected": -79.17742919921875,
      "loss": 0.18721944093704224,
      "memory(GiB)": 46.38,
      "nll_loss": 0.124346524477005,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3631533086299896,
      "rewards/margins": 6.952091693878174,
      "rewards/rejected": -6.5889387130737305,
      "step": 622,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.063217470233328,
      "grad_norm": 1.0911006927490234,
      "learning_rate": 9.759667268388191e-05,
      "logits/chosen": -1.3171772956848145,
      "logits/rejected": -1.3283257484436035,
      "logps/chosen": -6.717372417449951,
      "logps/rejected": -76.8449478149414,
      "loss": 0.20228637754917145,
      "memory(GiB)": 46.38,
      "nll_loss": 0.18503551185131073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.44489482045173645,
      "rewards/margins": 6.900110721588135,
      "rewards/rejected": -6.455215930938721,
      "step": 623,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0649260505099045,
      "grad_norm": 2.274223804473877,
      "learning_rate": 9.731399272908465e-05,
      "logits/chosen": -1.3501414060592651,
      "logits/rejected": -1.355926513671875,
      "logps/chosen": -4.459833145141602,
      "logps/rejected": -66.5104751586914,
      "loss": 0.24065664410591125,
      "memory(GiB)": 46.38,
      "nll_loss": 0.16927620768547058,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3595888614654541,
      "rewards/margins": 6.001706123352051,
      "rewards/rejected": -5.642117500305176,
      "step": 624,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.066634630786481,
      "grad_norm": 3.2183289527893066,
      "learning_rate": 9.703133425152996e-05,
      "logits/chosen": -1.3518933057785034,
      "logits/rejected": -1.3583866357803345,
      "logps/chosen": -6.575954437255859,
      "logps/rejected": -68.98614501953125,
      "loss": 0.26166409254074097,
      "memory(GiB)": 46.38,
      "nll_loss": 0.22645416855812073,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.41138094663619995,
      "rewards/margins": 6.283568382263184,
      "rewards/rejected": -5.872187614440918,
      "step": 625,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0683432110630573,
      "grad_norm": 1.71477472782135,
      "learning_rate": 9.674869951134752e-05,
      "logits/chosen": -1.3658617734909058,
      "logits/rejected": -1.3735222816467285,
      "logps/chosen": -5.780742645263672,
      "logps/rejected": -67.09764099121094,
      "loss": 0.3838789463043213,
      "memory(GiB)": 46.38,
      "nll_loss": 0.31154492497444153,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.26302045583724976,
      "rewards/margins": 5.848612308502197,
      "rewards/rejected": -5.585592269897461,
      "step": 626,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0700517913396337,
      "grad_norm": 0.8326086401939392,
      "learning_rate": 9.646609076847712e-05,
      "logits/chosen": -1.3865187168121338,
      "logits/rejected": -1.4000904560089111,
      "logps/chosen": -3.629927635192871,
      "logps/rejected": -82.82337188720703,
      "loss": 0.1214153990149498,
      "memory(GiB)": 46.38,
      "nll_loss": 0.11697927117347717,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5431358814239502,
      "rewards/margins": 7.4620490074157715,
      "rewards/rejected": -6.918912887573242,
      "step": 627,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.07176037161621,
      "grad_norm": 1.6905492544174194,
      "learning_rate": 9.618351028265077e-05,
      "logits/chosen": -1.354899287223816,
      "logits/rejected": -1.3660712242126465,
      "logps/chosen": -5.028237819671631,
      "logps/rejected": -69.88750457763672,
      "loss": 0.33384251594543457,
      "memory(GiB)": 46.38,
      "nll_loss": 0.25687023997306824,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.41417601704597473,
      "rewards/margins": 5.83312463760376,
      "rewards/rejected": -5.418948173522949,
      "step": 628,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.0734689518927867,
      "grad_norm": 1.6832268238067627,
      "learning_rate": 9.590096031337453e-05,
      "logits/chosen": -1.3819124698638916,
      "logits/rejected": -1.3906022310256958,
      "logps/chosen": -6.349166393280029,
      "logps/rejected": -68.72492218017578,
      "loss": 0.4190691411495209,
      "memory(GiB)": 46.38,
      "nll_loss": 0.34377047419548035,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3247349262237549,
      "rewards/margins": 5.935654640197754,
      "rewards/rejected": -5.61091947555542,
      "step": 629,
      "train_speed(iter/s)": 0.005441
    },
    {
      "epoch": 1.075177532169363,
      "grad_norm": 13.337855339050293,
      "learning_rate": 9.561844311991037e-05,
      "logits/chosen": -1.3419092893600464,
      "logits/rejected": -1.358843445777893,
      "logps/chosen": -2.356278896331787,
      "logps/rejected": -77.90457153320312,
      "loss": 0.17338180541992188,
      "memory(GiB)": 46.38,
      "nll_loss": 0.13151712715625763,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.3676100969314575,
      "rewards/margins": 6.811404228210449,
      "rewards/rejected": -6.443794250488281,
      "step": 630,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0768861124459395,
      "grad_norm": 1.4632470607757568,
      "learning_rate": 9.533596096125825e-05,
      "logits/chosen": -1.365177035331726,
      "logits/rejected": -1.3824915885925293,
      "logps/chosen": -6.562123775482178,
      "logps/rejected": -80.6185531616211,
      "loss": 0.3083136975765228,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2978009879589081,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4310091435909271,
      "rewards/margins": 7.036831855773926,
      "rewards/rejected": -6.6058220863342285,
      "step": 631,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0785946927225158,
      "grad_norm": 1.0129234790802002,
      "learning_rate": 9.505351609613802e-05,
      "logits/chosen": -1.3373836278915405,
      "logits/rejected": -1.3479772806167603,
      "logps/chosen": -4.921580791473389,
      "logps/rejected": -73.02953338623047,
      "loss": 0.24630959331989288,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20439808070659637,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.2602986693382263,
      "rewards/margins": 6.579464435577393,
      "rewards/rejected": -6.3191657066345215,
      "step": 632,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0803032729990922,
      "grad_norm": 1.1055867671966553,
      "learning_rate": 9.477111078297129e-05,
      "logits/chosen": -1.4558181762695312,
      "logits/rejected": -1.4696996212005615,
      "logps/chosen": -10.137053489685059,
      "logps/rejected": -81.14447784423828,
      "loss": 0.2885958254337311,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2826201617717743,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5288646817207336,
      "rewards/margins": 7.191718101501465,
      "rewards/rejected": -6.662853240966797,
      "step": 633,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0820118532756688,
      "grad_norm": 2.9462742805480957,
      "learning_rate": 9.448874727986336e-05,
      "logits/chosen": -1.4031565189361572,
      "logits/rejected": -1.4253337383270264,
      "logps/chosen": -2.6883082389831543,
      "logps/rejected": -84.79672241210938,
      "loss": 0.1640852689743042,
      "memory(GiB)": 46.38,
      "nll_loss": 0.10880641639232635,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.2951303720474243,
      "rewards/margins": 7.397609710693359,
      "rewards/rejected": -7.102480411529541,
      "step": 634,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0837204335522452,
      "grad_norm": 1.1565788984298706,
      "learning_rate": 9.420642784458535e-05,
      "logits/chosen": -1.4201645851135254,
      "logits/rejected": -1.4284991025924683,
      "logps/chosen": -5.476447582244873,
      "logps/rejected": -62.978973388671875,
      "loss": 0.37195485830307007,
      "memory(GiB)": 46.38,
      "nll_loss": 0.27154970169067383,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.4178648591041565,
      "rewards/margins": 5.697319030761719,
      "rewards/rejected": -5.279453754425049,
      "step": 635,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.0854290138288216,
      "grad_norm": 4.277892589569092,
      "learning_rate": 9.392415473455594e-05,
      "logits/chosen": -1.4194096326828003,
      "logits/rejected": -1.4362523555755615,
      "logps/chosen": -5.9199018478393555,
      "logps/rejected": -84.28883361816406,
      "loss": 0.2027808129787445,
      "memory(GiB)": 46.38,
      "nll_loss": 0.1775105595588684,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5387552976608276,
      "rewards/margins": 7.33411979675293,
      "rewards/rejected": -6.7953643798828125,
      "step": 636,
      "train_speed(iter/s)": 0.005442
    },
    {
      "epoch": 1.087137594105398,
      "grad_norm": 2.5401055812835693,
      "learning_rate": 9.364193020682332e-05,
      "logits/chosen": -1.4070954322814941,
      "logits/rejected": -1.4216212034225464,
      "logps/chosen": -3.7252540588378906,
      "logps/rejected": -84.25828552246094,
      "loss": 0.14458607137203217,
      "memory(GiB)": 46.38,
      "nll_loss": 0.13682252168655396,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3488786220550537,
      "rewards/margins": 7.642581462860107,
      "rewards/rejected": -7.293702602386475,
      "step": 637,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.0888461743819744,
      "grad_norm": 4.585189342498779,
      "learning_rate": 9.335975651804734e-05,
      "logits/chosen": -1.4299910068511963,
      "logits/rejected": -1.4420251846313477,
      "logps/chosen": -6.502732753753662,
      "logps/rejected": -73.0892562866211,
      "loss": 0.3148469924926758,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2556683123111725,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6074526309967041,
      "rewards/margins": 6.673343658447266,
      "rewards/rejected": -6.065891265869141,
      "step": 638,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.090554754658551,
      "grad_norm": 0.8174660205841064,
      "learning_rate": 9.30776359244813e-05,
      "logits/chosen": -1.439100980758667,
      "logits/rejected": -1.4511916637420654,
      "logps/chosen": -7.758155822753906,
      "logps/rejected": -85.3901596069336,
      "loss": 0.30996763706207275,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2977246046066284,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.6248569488525391,
      "rewards/margins": 7.353357315063477,
      "rewards/rejected": -6.728500843048096,
      "step": 639,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.0922633349351274,
      "grad_norm": 21.460296630859375,
      "learning_rate": 9.279557068195399e-05,
      "logits/chosen": -1.461799144744873,
      "logits/rejected": -1.4757378101348877,
      "logps/chosen": -6.172895908355713,
      "logps/rejected": -81.267822265625,
      "loss": 0.26451700925827026,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2501959204673767,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5207919478416443,
      "rewards/margins": 7.376910209655762,
      "rewards/rejected": -6.856119155883789,
      "step": 640,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.0939719152117038,
      "grad_norm": 1.6023881435394287,
      "learning_rate": 9.251356304585148e-05,
      "logits/chosen": -1.4143339395523071,
      "logits/rejected": -1.436486840248108,
      "logps/chosen": -6.017026901245117,
      "logps/rejected": -84.00882720947266,
      "loss": 0.3240937292575836,
      "memory(GiB)": 46.38,
      "nll_loss": 0.29061558842658997,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.30444955825805664,
      "rewards/margins": 7.431375503540039,
      "rewards/rejected": -7.126925945281982,
      "step": 641,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.0956804954882802,
      "grad_norm": 0.8944610953330994,
      "learning_rate": 9.223161527109937e-05,
      "logits/chosen": -1.4260931015014648,
      "logits/rejected": -1.447216272354126,
      "logps/chosen": -4.132955551147461,
      "logps/rejected": -88.57144927978516,
      "loss": 0.2163669317960739,
      "memory(GiB)": 46.38,
      "nll_loss": 0.19940386712551117,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.3151531219482422,
      "rewards/margins": 7.782842636108398,
      "rewards/rejected": -7.467689514160156,
      "step": 642,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.0973890757648566,
      "grad_norm": 1.0704683065414429,
      "learning_rate": 9.194972961214454e-05,
      "logits/chosen": -1.4121977090835571,
      "logits/rejected": -1.4206119775772095,
      "logps/chosen": -3.7283246517181396,
      "logps/rejected": -66.90413665771484,
      "loss": 0.2594975531101227,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20128358900547028,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.311231404542923,
      "rewards/margins": 5.829047679901123,
      "rewards/rejected": -5.517816543579102,
      "step": 643,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.0990976560414332,
      "grad_norm": 1.224323034286499,
      "learning_rate": 9.166790832293727e-05,
      "logits/chosen": -1.4135029315948486,
      "logits/rejected": -1.4257118701934814,
      "logps/chosen": -6.872603893280029,
      "logps/rejected": -64.60710144042969,
      "loss": 0.296747088432312,
      "memory(GiB)": 46.38,
      "nll_loss": 0.22850407660007477,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5369666814804077,
      "rewards/margins": 5.512694835662842,
      "rewards/rejected": -4.9757280349731445,
      "step": 644,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.1008062363180096,
      "grad_norm": 0.9509670734405518,
      "learning_rate": 9.138615365691297e-05,
      "logits/chosen": -1.4065271615982056,
      "logits/rejected": -1.4219188690185547,
      "logps/chosen": -6.971405029296875,
      "logps/rejected": -77.34842681884766,
      "loss": 0.25675371289253235,
      "memory(GiB)": 46.38,
      "nll_loss": 0.24975880980491638,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.28990283608436584,
      "rewards/margins": 6.457014083862305,
      "rewards/rejected": -6.167111873626709,
      "step": 645,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.102514816594586,
      "grad_norm": 1.9502674341201782,
      "learning_rate": 9.110446786697449e-05,
      "logits/chosen": -1.3356096744537354,
      "logits/rejected": -1.3449259996414185,
      "logps/chosen": -7.475038528442383,
      "logps/rejected": -75.35752868652344,
      "loss": 0.28363364934921265,
      "memory(GiB)": 46.38,
      "nll_loss": 0.2586205303668976,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5652132630348206,
      "rewards/margins": 6.60277795791626,
      "rewards/rejected": -6.037565231323242,
      "step": 646,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.1042233968711623,
      "grad_norm": 1.578850269317627,
      "learning_rate": 9.082285320547396e-05,
      "logits/chosen": -1.3386077880859375,
      "logits/rejected": -1.3520487546920776,
      "logps/chosen": -4.563968658447266,
      "logps/rejected": -79.4625015258789,
      "loss": 0.2268017679452896,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20012691617012024,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.47328507900238037,
      "rewards/margins": 7.045524597167969,
      "rewards/rejected": -6.572238922119141,
      "step": 647,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.1059319771477387,
      "grad_norm": 1.0217214822769165,
      "learning_rate": 9.054131192419459e-05,
      "logits/chosen": -1.3284649848937988,
      "logits/rejected": -1.3363628387451172,
      "logps/chosen": -4.5251946449279785,
      "logps/rejected": -66.22467041015625,
      "loss": 0.278721421957016,
      "memory(GiB)": 46.38,
      "nll_loss": 0.20027558505535126,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": 0.6165156364440918,
      "rewards/margins": 6.0968852043151855,
      "rewards/rejected": -5.480369567871094,
      "step": 648,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.1076405574243153,
      "grad_norm": 9.55971908569336,
      "learning_rate": 9.025984627433303e-05,
      "logits/chosen": -1.2837365865707397,
      "logits/rejected": -1.2869882583618164,
      "logps/chosen": -6.292945861816406,
      "logps/rejected": -65.33118438720703,
      "loss": 0.3155813217163086,
      "memory(GiB)": 46.38,
      "nll_loss": 0.26425838470458984,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.5627895593643188,
      "rewards/margins": 5.7191643714904785,
      "rewards/rejected": -5.156374931335449,
      "step": 649,
      "train_speed(iter/s)": 0.005443
    },
    {
      "epoch": 1.1093491377008917,
      "grad_norm": 0.9349164366722107,
      "learning_rate": 8.997845850648114e-05,
      "logits/chosen": -1.2805975675582886,
      "logits/rejected": -1.2881710529327393,
      "logps/chosen": -4.299163818359375,
      "logps/rejected": -71.964599609375,
      "loss": 0.23460139334201813,
      "memory(GiB)": 46.38,
      "nll_loss": 0.21636176109313965,
      "rewards/accuracies": 1.0,
      "rewards/chosen": 0.4982881546020508,
      "rewards/margins": 6.471682071685791,
      "rewards/rejected": -5.97339391708374,
      "step": 650,
      "train_speed(iter/s)": 0.005443
    }
  ],
  "logging_steps": 1,
  "max_steps": 1170,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 4.734511367954891e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}