{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.44742729306487694,
  "eval_steps": 300,
  "global_step": 100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.0044742729306487695,
      "grad_norm": 4.521431922912598,
      "learning_rate": 8.695652173913044e-06,
      "logits/chosen": -0.37893733382225037,
      "logits/rejected": -0.4597730040550232,
      "logps/chosen": -11.001541137695312,
      "logps/rejected": -17.05784034729004,
      "loss": 1.3586950302124023,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6655479669570923,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 1,
      "train_speed(iter/s)": 0.005374
    },
    {
      "epoch": 0.008948545861297539,
      "grad_norm": 3.470855951309204,
      "learning_rate": 1.739130434782609e-05,
      "logits/chosen": -0.3633067309856415,
      "logits/rejected": -0.4392029941082001,
      "logps/chosen": -7.973292350769043,
      "logps/rejected": -13.90955924987793,
      "loss": 1.2437297105789185,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5505824685096741,
      "rewards/accuracies": 0.0,
      "rewards/chosen": 0.0,
      "rewards/margins": 0.0,
      "rewards/rejected": 0.0,
      "step": 2,
      "train_speed(iter/s)": 0.005501
    },
    {
      "epoch": 0.013422818791946308,
      "grad_norm": 5.51271390914917,
      "learning_rate": 2.608695652173913e-05,
      "logits/chosen": -0.26578277349472046,
      "logits/rejected": -0.45589232444763184,
      "logps/chosen": -9.600592613220215,
      "logps/rejected": -22.355058670043945,
      "loss": 1.392338514328003,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6997992396354675,
      "rewards/accuracies": 0.453125,
      "rewards/chosen": 4.354381235316396e-05,
      "rewards/margins": 0.0013819998130202293,
      "rewards/rejected": -0.001338455593213439,
      "step": 3,
      "train_speed(iter/s)": 0.005521
    },
    {
      "epoch": 0.017897091722595078,
      "grad_norm": 6.5741682052612305,
      "learning_rate": 3.478260869565218e-05,
      "logits/chosen": -0.40403854846954346,
      "logits/rejected": -0.5402908325195312,
      "logps/chosen": -6.373080253601074,
      "logps/rejected": -16.791393280029297,
      "loss": 1.222740888595581,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5450983047485352,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.010452846065163612,
      "rewards/margins": 0.031955912709236145,
      "rewards/rejected": -0.021503068506717682,
      "step": 4,
      "train_speed(iter/s)": 0.005551
    },
    {
      "epoch": 0.02237136465324385,
      "grad_norm": 3.046576738357544,
      "learning_rate": 4.347826086956522e-05,
      "logits/chosen": -0.3187038004398346,
      "logits/rejected": -0.4441537857055664,
      "logps/chosen": -9.070188522338867,
      "logps/rejected": -17.255901336669922,
      "loss": 1.2276911735534668,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5655408501625061,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.007649715524166822,
      "rewards/margins": 0.07260076701641083,
      "rewards/rejected": -0.08025049418210983,
      "step": 5,
      "train_speed(iter/s)": 0.005557
    },
    {
      "epoch": 0.026845637583892617,
      "grad_norm": 4.846445560455322,
      "learning_rate": 5.217391304347826e-05,
      "logits/chosen": -0.30176016688346863,
      "logits/rejected": -0.45638489723205566,
      "logps/chosen": -8.63841438293457,
      "logps/rejected": -18.97266960144043,
      "loss": 1.1484646797180176,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5371410250663757,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.024880727753043175,
      "rewards/margins": 0.23170879483222961,
      "rewards/rejected": -0.25658950209617615,
      "step": 6,
      "train_speed(iter/s)": 0.005568
    },
    {
      "epoch": 0.03131991051454139,
      "grad_norm": 7.562114715576172,
      "learning_rate": 6.086956521739131e-05,
      "logits/chosen": -0.3937205672264099,
      "logits/rejected": -0.48791196942329407,
      "logps/chosen": -10.315498352050781,
      "logps/rejected": -20.046728134155273,
      "loss": 1.3093780279159546,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6873191595077515,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.14371810853481293,
      "rewards/margins": 0.2797008454799652,
      "rewards/rejected": -0.42341893911361694,
      "step": 7,
      "train_speed(iter/s)": 0.005577
    },
    {
      "epoch": 0.035794183445190156,
      "grad_norm": 7.459152698516846,
      "learning_rate": 6.956521739130436e-05,
      "logits/chosen": -0.3540106415748596,
      "logits/rejected": -0.43836480379104614,
      "logps/chosen": -8.758136749267578,
      "logps/rejected": -18.294340133666992,
      "loss": 1.3215807676315308,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6817622184753418,
      "rewards/accuracies": 0.640625,
      "rewards/chosen": -0.19053705036640167,
      "rewards/margins": 0.2214164435863495,
      "rewards/rejected": -0.41195347905158997,
      "step": 8,
      "train_speed(iter/s)": 0.005585
    },
    {
      "epoch": 0.040268456375838924,
      "grad_norm": 3.0581202507019043,
      "learning_rate": 7.82608695652174e-05,
      "logits/chosen": -0.3466319143772125,
      "logits/rejected": -0.4212740361690521,
      "logps/chosen": -10.387115478515625,
      "logps/rejected": -20.363929748535156,
      "loss": 1.1448516845703125,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5626436471939087,
      "rewards/accuracies": 0.703125,
      "rewards/chosen": -0.06380077451467514,
      "rewards/margins": 0.3404861092567444,
      "rewards/rejected": -0.40428686141967773,
      "step": 9,
      "train_speed(iter/s)": 0.005583
    },
    {
      "epoch": 0.0447427293064877,
      "grad_norm": 3.9353108406066895,
      "learning_rate": 8.695652173913044e-05,
      "logits/chosen": -0.4188399314880371,
      "logits/rejected": -0.5007613301277161,
      "logps/chosen": -9.096799850463867,
      "logps/rejected": -18.05609130859375,
      "loss": 1.2927519083023071,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6729815006256104,
      "rewards/accuracies": 0.625,
      "rewards/chosen": -0.060015156865119934,
      "rewards/margins": 0.20498159527778625,
      "rewards/rejected": -0.264996737241745,
      "step": 10,
      "train_speed(iter/s)": 0.00559
    },
    {
      "epoch": 0.049217002237136466,
      "grad_norm": 2.2350685596466064,
      "learning_rate": 9.565217391304348e-05,
      "logits/chosen": -0.41371461749076843,
      "logits/rejected": -0.4829203188419342,
      "logps/chosen": -7.282920837402344,
      "logps/rejected": -15.015157699584961,
      "loss": 1.167080044746399,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5672391653060913,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.07586651295423508,
      "rewards/margins": 0.2256288230419159,
      "rewards/rejected": -0.14976228773593903,
      "step": 11,
      "train_speed(iter/s)": 0.005596
    },
    {
      "epoch": 0.053691275167785234,
      "grad_norm": 2.6733052730560303,
      "learning_rate": 0.00010434782608695653,
      "logits/chosen": -0.38637417554855347,
      "logits/rejected": -0.45029905438423157,
      "logps/chosen": -11.042367935180664,
      "logps/rejected": -16.67047882080078,
      "loss": 1.2763197422027588,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6531897783279419,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": 0.06360332667827606,
      "rewards/margins": 0.17586028575897217,
      "rewards/rejected": -0.1122569665312767,
      "step": 12,
      "train_speed(iter/s)": 0.005598
    },
    {
      "epoch": 0.058165548098434,
      "grad_norm": 1.9472479820251465,
      "learning_rate": 0.00011304347826086956,
      "logits/chosen": -0.4195230007171631,
      "logits/rejected": -0.4734322428703308,
      "logps/chosen": -7.205465316772461,
      "logps/rejected": -13.222125053405762,
      "loss": 1.1376935243606567,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4900382459163666,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.018632210791110992,
      "rewards/margins": 0.11014734208583832,
      "rewards/rejected": -0.09151513874530792,
      "step": 13,
      "train_speed(iter/s)": 0.005601
    },
    {
      "epoch": 0.06263982102908278,
      "grad_norm": 1.828997015953064,
      "learning_rate": 0.00012173913043478263,
      "logits/chosen": -0.399485319852829,
      "logits/rejected": -0.4682197570800781,
      "logps/chosen": -8.005407333374023,
      "logps/rejected": -16.06272315979004,
      "loss": 1.1697134971618652,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5445835590362549,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0030609495006501675,
      "rewards/margins": 0.16438230872154236,
      "rewards/rejected": -0.16132137179374695,
      "step": 14,
      "train_speed(iter/s)": 0.005604
    },
    {
      "epoch": 0.06711409395973154,
      "grad_norm": 2.1229512691497803,
      "learning_rate": 0.00013043478260869567,
      "logits/chosen": -0.3835911750793457,
      "logits/rejected": -0.4592927396297455,
      "logps/chosen": -6.499156475067139,
      "logps/rejected": -15.707530975341797,
      "loss": 1.0722893476486206,
      "memory(GiB)": 46.57,
      "nll_loss": 0.503006637096405,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06762155145406723,
      "rewards/margins": 0.31041574478149414,
      "rewards/rejected": -0.24279415607452393,
      "step": 15,
      "train_speed(iter/s)": 0.005608
    },
    {
      "epoch": 0.07158836689038031,
      "grad_norm": 4.802647113800049,
      "learning_rate": 0.0001391304347826087,
      "logits/chosen": -0.3306896388530731,
      "logits/rejected": -0.41153809428215027,
      "logps/chosen": -10.432010650634766,
      "logps/rejected": -21.58324432373047,
      "loss": 1.3572790622711182,
      "memory(GiB)": 46.57,
      "nll_loss": 0.7762681245803833,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.06817664206027985,
      "rewards/margins": 0.363118052482605,
      "rewards/rejected": -0.4312947690486908,
      "step": 16,
      "train_speed(iter/s)": 0.005608
    },
    {
      "epoch": 0.07606263982102908,
      "grad_norm": 4.122814178466797,
      "learning_rate": 0.00014782608695652173,
      "logits/chosen": -0.32593876123428345,
      "logits/rejected": -0.4176383316516876,
      "logps/chosen": -9.068134307861328,
      "logps/rejected": -21.460546493530273,
      "loss": 1.1545155048370361,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5955549478530884,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0028565023094415665,
      "rewards/margins": 0.3992985486984253,
      "rewards/rejected": -0.3964419960975647,
      "step": 17,
      "train_speed(iter/s)": 0.00561
    },
    {
      "epoch": 0.08053691275167785,
      "grad_norm": 1.9508388042449951,
      "learning_rate": 0.0001565217391304348,
      "logits/chosen": -0.3363662660121918,
      "logits/rejected": -0.40299132466316223,
      "logps/chosen": -7.306541919708252,
      "logps/rejected": -15.145895004272461,
      "loss": 1.1673789024353027,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5968848466873169,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13542409241199493,
      "rewards/margins": 0.3165813684463501,
      "rewards/rejected": -0.18115724623203278,
      "step": 18,
      "train_speed(iter/s)": 0.005611
    },
    {
      "epoch": 0.08501118568232663,
      "grad_norm": 4.076289176940918,
      "learning_rate": 0.00016521739130434784,
      "logits/chosen": -0.33677053451538086,
      "logits/rejected": -0.42119455337524414,
      "logps/chosen": -6.263703346252441,
      "logps/rejected": -17.470884323120117,
      "loss": 0.9588873982429504,
      "memory(GiB)": 46.57,
      "nll_loss": 0.42532750964164734,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.17247594892978668,
      "rewards/margins": 0.41344738006591797,
      "rewards/rejected": -0.24097144603729248,
      "step": 19,
      "train_speed(iter/s)": 0.005613
    },
    {
      "epoch": 0.0894854586129754,
      "grad_norm": 2.8480069637298584,
      "learning_rate": 0.00017391304347826088,
      "logits/chosen": -0.27281832695007324,
      "logits/rejected": -0.3382987976074219,
      "logps/chosen": -8.123124122619629,
      "logps/rejected": -17.66653060913086,
      "loss": 1.1668881177902222,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5924651622772217,
      "rewards/accuracies": 0.71875,
      "rewards/chosen": -0.00576989259570837,
      "rewards/margins": 0.35684970021247864,
      "rewards/rejected": -0.36261963844299316,
      "step": 20,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.09395973154362416,
      "grad_norm": 2.664872646331787,
      "learning_rate": 0.00018260869565217392,
      "logits/chosen": -0.2556913197040558,
      "logits/rejected": -0.3474789261817932,
      "logps/chosen": -5.220917701721191,
      "logps/rejected": -23.547605514526367,
      "loss": 0.9417474865913391,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4541093111038208,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.043988656252622604,
      "rewards/margins": 0.6349199414253235,
      "rewards/rejected": -0.5909312963485718,
      "step": 21,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.09843400447427293,
      "grad_norm": 2.325854539871216,
      "learning_rate": 0.00019130434782608697,
      "logits/chosen": -0.2636699676513672,
      "logits/rejected": -0.33711570501327515,
      "logps/chosen": -8.976820945739746,
      "logps/rejected": -19.829130172729492,
      "loss": 1.0729048252105713,
      "memory(GiB)": 46.57,
      "nll_loss": 0.595395028591156,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09026271104812622,
      "rewards/margins": 0.623466432094574,
      "rewards/rejected": -0.5332037210464478,
      "step": 22,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.1029082774049217,
      "grad_norm": 2.684507369995117,
      "learning_rate": 0.0002,
      "logits/chosen": -0.24679847061634064,
      "logits/rejected": -0.3378849923610687,
      "logps/chosen": -7.14666223526001,
      "logps/rejected": -23.644922256469727,
      "loss": 0.9377666115760803,
      "memory(GiB)": 46.57,
      "nll_loss": 0.427379846572876,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.0773518830537796,
      "rewards/margins": 0.6581345796585083,
      "rewards/rejected": -0.7354865074157715,
      "step": 23,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.10738255033557047,
      "grad_norm": 2.6381781101226807,
      "learning_rate": 0.00019999724204599747,
      "logits/chosen": -0.246232807636261,
      "logits/rejected": -0.3716464936733246,
      "logps/chosen": -6.111550331115723,
      "logps/rejected": -24.672321319580078,
      "loss": 0.9850844740867615,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5655074119567871,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.027247514575719833,
      "rewards/margins": 0.8617139458656311,
      "rewards/rejected": -0.8344664573669434,
      "step": 24,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.11185682326621924,
      "grad_norm": 2.5178840160369873,
      "learning_rate": 0.00019998896833611603,
      "logits/chosen": -0.23005428910255432,
      "logits/rejected": -0.3447636365890503,
      "logps/chosen": -8.92835807800293,
      "logps/rejected": -25.173357009887695,
      "loss": 1.1135735511779785,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5982357263565063,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.07063186913728714,
      "rewards/margins": 0.6180722117424011,
      "rewards/rejected": -0.6887040138244629,
      "step": 25,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.116331096196868,
      "grad_norm": 1.9486284255981445,
      "learning_rate": 0.0001999751793267259,
      "logits/chosen": -0.17476646602153778,
      "logits/rejected": -0.28910771012306213,
      "logps/chosen": -6.982065200805664,
      "logps/rejected": -25.235271453857422,
      "loss": 0.9616149067878723,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4814175069332123,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.017612511292099953,
      "rewards/margins": 0.7644082307815552,
      "rewards/rejected": -0.7467955350875854,
      "step": 26,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.12080536912751678,
      "grad_norm": 2.024102210998535,
      "learning_rate": 0.0001999558757784162,
      "logits/chosen": -0.22182828187942505,
      "logits/rejected": -0.33474183082580566,
      "logps/chosen": -8.283013343811035,
      "logps/rejected": -23.36388397216797,
      "loss": 1.071007251739502,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5651244521141052,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.005644451826810837,
      "rewards/margins": 0.549856424331665,
      "rewards/rejected": -0.5555009245872498,
      "step": 27,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.12527964205816555,
      "grad_norm": 2.334916830062866,
      "learning_rate": 0.0001999310587559529,
      "logits/chosen": -0.1745743602514267,
      "logits/rejected": -0.28861111402511597,
      "logps/chosen": -7.148108959197998,
      "logps/rejected": -22.50945472717285,
      "loss": 1.0254650115966797,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5092177391052246,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.024923313409090042,
      "rewards/margins": 0.6763253211975098,
      "rewards/rejected": -0.6514019966125488,
      "step": 28,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.1297539149888143,
      "grad_norm": 4.392460823059082,
      "learning_rate": 0.00019990072962822007,
      "logits/chosen": -0.1607450246810913,
      "logits/rejected": -0.30258849263191223,
      "logps/chosen": -8.330341339111328,
      "logps/rejected": -23.302955627441406,
      "loss": 1.1018993854522705,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6234543919563293,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1296255737543106,
      "rewards/margins": 0.7057831883430481,
      "rewards/rejected": -0.5761575102806091,
      "step": 29,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.1342281879194631,
      "grad_norm": 1.9557713270187378,
      "learning_rate": 0.00019986489006814452,
      "logits/chosen": -0.15406475961208344,
      "logits/rejected": -0.32140052318573,
      "logps/chosen": -6.338605880737305,
      "logps/rejected": -27.934860229492188,
      "loss": 0.8741156458854675,
      "memory(GiB)": 46.57,
      "nll_loss": 0.43781277537345886,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.005505967885255814,
      "rewards/margins": 0.9152470827102661,
      "rewards/rejected": -0.9097411632537842,
      "step": 30,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.13870246085011187,
      "grad_norm": 3.570540189743042,
      "learning_rate": 0.00019982354205260347,
      "logits/chosen": -0.13595342636108398,
      "logits/rejected": -0.24140222370624542,
      "logps/chosen": -9.938725471496582,
      "logps/rejected": -26.17535972595215,
      "loss": 1.1157177686691284,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6451507806777954,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06947211921215057,
      "rewards/margins": 0.9398424029350281,
      "rewards/rejected": -1.0093145370483398,
      "step": 31,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.14317673378076062,
      "grad_norm": 2.263465404510498,
      "learning_rate": 0.00019977668786231534,
      "logits/chosen": -0.13489112257957458,
      "logits/rejected": -0.2546093165874481,
      "logps/chosen": -9.485723495483398,
      "logps/rejected": -30.659465789794922,
      "loss": 1.1024084091186523,
      "memory(GiB)": 46.57,
      "nll_loss": 0.7035719752311707,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.094739630818367,
      "rewards/margins": 1.1513433456420898,
      "rewards/rejected": -1.0566036701202393,
      "step": 32,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.1476510067114094,
      "grad_norm": 4.003927230834961,
      "learning_rate": 0.00019972433008171416,
      "logits/chosen": -0.2770022451877594,
      "logits/rejected": -0.3445180356502533,
      "logps/chosen": -7.854065895080566,
      "logps/rejected": -17.47032356262207,
      "loss": 1.1709215641021729,
      "memory(GiB)": 46.57,
      "nll_loss": 0.635529637336731,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.03749306499958038,
      "rewards/margins": 0.6043045520782471,
      "rewards/rejected": -0.5668113827705383,
      "step": 33,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.15212527964205816,
      "grad_norm": 2.930697202682495,
      "learning_rate": 0.00019966647159880703,
      "logits/chosen": -0.20737791061401367,
      "logits/rejected": -0.3068307638168335,
      "logps/chosen": -5.924644470214844,
      "logps/rejected": -20.47597885131836,
      "loss": 0.9971151947975159,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4416566789150238,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.04966810718178749,
      "rewards/margins": 0.48412013053894043,
      "rewards/rejected": -0.4344520568847656,
      "step": 34,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.15659955257270694,
      "grad_norm": 2.292167901992798,
      "learning_rate": 0.00019960311560501454,
      "logits/chosen": -0.20404092967510223,
      "logits/rejected": -0.2819429636001587,
      "logps/chosen": -9.679859161376953,
      "logps/rejected": -22.75543785095215,
      "loss": 1.1032215356826782,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6184647679328918,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07092233002185822,
      "rewards/margins": 0.8317320942878723,
      "rewards/rejected": -0.7608097791671753,
      "step": 35,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.1610738255033557,
      "grad_norm": 1.5171147584915161,
      "learning_rate": 0.0001995342655949951,
      "logits/chosen": -0.18677955865859985,
      "logits/rejected": -0.30433592200279236,
      "logps/chosen": -7.317014217376709,
      "logps/rejected": -23.58348274230957,
      "loss": 1.0045303106307983,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5375936031341553,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.13633473217487335,
      "rewards/margins": 0.7853459119796753,
      "rewards/rejected": -0.6490110754966736,
      "step": 36,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.16554809843400448,
      "grad_norm": 1.8760979175567627,
      "learning_rate": 0.00019945992536645187,
      "logits/chosen": -0.17943330109119415,
      "logits/rejected": -0.27492690086364746,
      "logps/chosen": -9.596693992614746,
      "logps/rejected": -25.057254791259766,
      "loss": 1.0518603324890137,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5895128846168518,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 4.468485713005066e-06,
      "rewards/margins": 0.8271102905273438,
      "rewards/rejected": -0.827105700969696,
      "step": 37,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.17002237136465326,
      "grad_norm": 2.1765875816345215,
      "learning_rate": 0.0001993800990199235,
      "logits/chosen": -0.17796888947486877,
      "logits/rejected": -0.27494707703590393,
      "logps/chosen": -8.854700088500977,
      "logps/rejected": -22.589916229248047,
      "loss": 1.1736024618148804,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6680553555488586,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.027395736426115036,
      "rewards/margins": 0.702995777130127,
      "rewards/rejected": -0.6756000518798828,
      "step": 38,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.174496644295302,
      "grad_norm": 2.0398788452148438,
      "learning_rate": 0.0001992947909585578,
      "logits/chosen": -0.15950845181941986,
      "logits/rejected": -0.285598486661911,
      "logps/chosen": -7.358231544494629,
      "logps/rejected": -23.962072372436523,
      "loss": 0.9235795736312866,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5008021593093872,
      "rewards/accuracies": 0.890625,
      "rewards/chosen": 0.1876223385334015,
      "rewards/margins": 0.9989414215087891,
      "rewards/rejected": -0.8113190531730652,
      "step": 39,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.1789709172259508,
      "grad_norm": 2.137016773223877,
      "learning_rate": 0.000199204005887869,
      "logits/chosen": -0.18114963173866272,
      "logits/rejected": -0.23074296116828918,
      "logps/chosen": -9.713254928588867,
      "logps/rejected": -15.0142822265625,
      "loss": 1.1612474918365479,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5743556618690491,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.08143825083971024,
      "rewards/margins": 0.3889071047306061,
      "rewards/rejected": -0.30746886134147644,
      "step": 40,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.18344519015659955,
      "grad_norm": 2.5089111328125,
      "learning_rate": 0.000199107748815478,
      "logits/chosen": -0.17036814987659454,
      "logits/rejected": -0.25284841656684875,
      "logps/chosen": -7.5790181159973145,
      "logps/rejected": -22.757585525512695,
      "loss": 0.9598883986473083,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5350608229637146,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.10258667171001434,
      "rewards/margins": 0.982707679271698,
      "rewards/rejected": -0.8801209926605225,
      "step": 41,
      "train_speed(iter/s)": 0.005622
    },
    {
      "epoch": 0.18791946308724833,
      "grad_norm": 2.425978660583496,
      "learning_rate": 0.00019900602505083648,
      "logits/chosen": -0.17056672275066376,
      "logits/rejected": -0.24402382969856262,
      "logps/chosen": -9.11720085144043,
      "logps/rejected": -26.321962356567383,
      "loss": 0.9588575959205627,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5199980735778809,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": -0.02633209154009819,
      "rewards/margins": 1.148821234703064,
      "rewards/rejected": -1.1751532554626465,
      "step": 42,
      "train_speed(iter/s)": 0.005622
    },
    {
      "epoch": 0.19239373601789708,
      "grad_norm": 4.055136203765869,
      "learning_rate": 0.0001988988402049336,
      "logits/chosen": -0.06654427945613861,
      "logits/rejected": -0.19404850900173187,
      "logps/chosen": -10.976348876953125,
      "logps/rejected": -35.80363464355469,
      "loss": 1.205069661140442,
      "memory(GiB)": 46.57,
      "nll_loss": 0.7504922151565552,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.11243688315153122,
      "rewards/margins": 1.5014084577560425,
      "rewards/rejected": -1.6138453483581543,
      "step": 43,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.19686800894854586,
      "grad_norm": 2.9351325035095215,
      "learning_rate": 0.00019878620018998696,
      "logits/chosen": -0.21978600323200226,
      "logits/rejected": -0.32882797718048096,
      "logps/chosen": -6.976373195648193,
      "logps/rejected": -30.215131759643555,
      "loss": 0.9490640759468079,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5300356149673462,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": -0.11029693484306335,
      "rewards/margins": 1.1996657848358154,
      "rewards/rejected": -1.3099626302719116,
      "step": 44,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.20134228187919462,
      "grad_norm": 2.6874403953552246,
      "learning_rate": 0.00019866811121911607,
      "logits/chosen": -0.1936551034450531,
      "logits/rejected": -0.284149706363678,
      "logps/chosen": -6.9035515785217285,
      "logps/rejected": -22.284351348876953,
      "loss": 0.8805959820747375,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4068413972854614,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.07986758649349213,
      "rewards/margins": 0.8386067152023315,
      "rewards/rejected": -0.7587391138076782,
      "step": 45,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.2058165548098434,
      "grad_norm": 1.7893315553665161,
      "learning_rate": 0.000198544579806,
      "logits/chosen": -0.20961534976959229,
      "logits/rejected": -0.33521464467048645,
      "logps/chosen": -7.002056121826172,
      "logps/rejected": -25.93248748779297,
      "loss": 0.9666993618011475,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4805544912815094,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.07972076535224915,
      "rewards/margins": 0.8979825377464294,
      "rewards/rejected": -0.8182618021965027,
      "step": 46,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.21029082774049218,
      "grad_norm": 2.2033731937408447,
      "learning_rate": 0.0001984156127645178,
      "logits/chosen": -0.21569889783859253,
      "logits/rejected": -0.38949355483055115,
      "logps/chosen": -5.476630687713623,
      "logps/rejected": -27.572505950927734,
      "loss": 0.8785698413848877,
      "memory(GiB)": 46.57,
      "nll_loss": 0.42979833483695984,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.17229261994361877,
      "rewards/margins": 1.044838547706604,
      "rewards/rejected": -0.8725459575653076,
      "step": 47,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.21476510067114093,
      "grad_norm": 2.1163032054901123,
      "learning_rate": 0.00019828121720837286,
      "logits/chosen": -0.34833359718322754,
      "logits/rejected": -0.41568437218666077,
      "logps/chosen": -7.711272239685059,
      "logps/rejected": -25.60678482055664,
      "loss": 0.9890228509902954,
      "memory(GiB)": 46.57,
      "nll_loss": 0.548301100730896,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.07654333114624023,
      "rewards/margins": 1.2576009035110474,
      "rewards/rejected": -1.1810575723648071,
      "step": 48,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.21923937360178972,
      "grad_norm": 1.987210988998413,
      "learning_rate": 0.00019814140055070042,
      "logits/chosen": -0.2835751473903656,
      "logits/rejected": -0.38263481855392456,
      "logps/chosen": -8.624626159667969,
      "logps/rejected": -25.244403839111328,
      "loss": 0.9971789121627808,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5687417387962341,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.09697108715772629,
      "rewards/margins": 1.0362679958343506,
      "rewards/rejected": -0.9392969608306885,
      "step": 49,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.22371364653243847,
      "grad_norm": 1.9501866102218628,
      "learning_rate": 0.0001979961705036587,
      "logits/chosen": -0.38833191990852356,
      "logits/rejected": -0.4256289303302765,
      "logps/chosen": -8.22580623626709,
      "logps/rejected": -23.093332290649414,
      "loss": 0.9298551082611084,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4849992096424103,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.04563959687948227,
      "rewards/margins": 1.0999000072479248,
      "rewards/rejected": -1.054260492324829,
      "step": 50,
      "train_speed(iter/s)": 0.005622
    },
    {
      "epoch": 0.22818791946308725,
      "grad_norm": 2.496234893798828,
      "learning_rate": 0.00019784553507800349,
      "logits/chosen": -0.2710403501987457,
      "logits/rejected": -0.41735631227493286,
      "logps/chosen": -8.894371032714844,
      "logps/rejected": -28.586936950683594,
      "loss": 1.1125377416610718,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6875916719436646,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.10749680548906326,
      "rewards/margins": 1.2817683219909668,
      "rewards/rejected": -1.174271583557129,
      "step": 51,
      "train_speed(iter/s)": 0.005611
    },
    {
      "epoch": 0.232662192393736,
      "grad_norm": 3.274833917617798,
      "learning_rate": 0.00019768950258264623,
      "logits/chosen": -0.39167118072509766,
      "logits/rejected": -0.4531523883342743,
      "logps/chosen": -9.231929779052734,
      "logps/rejected": -22.90334701538086,
      "loss": 1.1200237274169922,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6103911399841309,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.16369852423667908,
      "rewards/margins": 0.9332659244537354,
      "rewards/rejected": -1.0969644784927368,
      "step": 52,
      "train_speed(iter/s)": 0.005612
    },
    {
      "epoch": 0.2371364653243848,
      "grad_norm": 2.9111413955688477,
      "learning_rate": 0.0001975280816241959,
      "logits/chosen": -0.3340080976486206,
      "logits/rejected": -0.3975221514701843,
      "logps/chosen": -11.631303787231445,
      "logps/rejected": -26.26183319091797,
      "loss": 1.1326872110366821,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6245000958442688,
      "rewards/accuracies": 0.75,
      "rewards/chosen": -0.032076627016067505,
      "rewards/margins": 0.9003015160560608,
      "rewards/rejected": -0.9323781132698059,
      "step": 53,
      "train_speed(iter/s)": 0.005612
    },
    {
      "epoch": 0.24161073825503357,
      "grad_norm": 2.7828478813171387,
      "learning_rate": 0.00019736128110648407,
      "logits/chosen": -0.345708429813385,
      "logits/rejected": -0.447677880525589,
      "logps/chosen": -8.686080932617188,
      "logps/rejected": -23.45962142944336,
      "loss": 0.9978072047233582,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5644747018814087,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.18989036977291107,
      "rewards/margins": 1.1108869314193726,
      "rewards/rejected": -0.9209966659545898,
      "step": 54,
      "train_speed(iter/s)": 0.005613
    },
    {
      "epoch": 0.24608501118568232,
      "grad_norm": 2.1220943927764893,
      "learning_rate": 0.0001971891102300738,
      "logits/chosen": -0.3758760690689087,
      "logits/rejected": -0.4768344461917877,
      "logps/chosen": -7.041341304779053,
      "logps/rejected": -22.334056854248047,
      "loss": 1.0144665241241455,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5401796698570251,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.1189827248454094,
      "rewards/margins": 0.939737856388092,
      "rewards/rejected": -0.820755124092102,
      "step": 55,
      "train_speed(iter/s)": 0.005613
    },
    {
      "epoch": 0.2505592841163311,
      "grad_norm": 1.843038558959961,
      "learning_rate": 0.00019701157849175228,
      "logits/chosen": -0.37703099846839905,
      "logits/rejected": -0.46533748507499695,
      "logps/chosen": -8.012426376342773,
      "logps/rejected": -22.621742248535156,
      "loss": 1.0123478174209595,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5707098245620728,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.15585632622241974,
      "rewards/margins": 0.9487887620925903,
      "rewards/rejected": -0.792932391166687,
      "step": 56,
      "train_speed(iter/s)": 0.005614
    },
    {
      "epoch": 0.2550335570469799,
      "grad_norm": 2.230550527572632,
      "learning_rate": 0.00019682869568400684,
      "logits/chosen": -0.27077603340148926,
      "logits/rejected": -0.43189266324043274,
      "logps/chosen": -6.5583930015563965,
      "logps/rejected": -29.555484771728516,
      "loss": 0.9619537591934204,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5382556915283203,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.012836463749408722,
      "rewards/margins": 1.0998661518096924,
      "rewards/rejected": -1.0870296955108643,
      "step": 57,
      "train_speed(iter/s)": 0.005614
    },
    {
      "epoch": 0.2595078299776286,
      "grad_norm": 2.252734661102295,
      "learning_rate": 0.00019664047189448493,
      "logits/chosen": -0.30061304569244385,
      "logits/rejected": -0.4390646517276764,
      "logps/chosen": -5.5507659912109375,
      "logps/rejected": -26.51093292236328,
      "loss": 0.9082356095314026,
      "memory(GiB)": 46.57,
      "nll_loss": 0.41620010137557983,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.027434132993221283,
      "rewards/margins": 0.9383218288421631,
      "rewards/rejected": -0.9657559394836426,
      "step": 58,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.2639821029082774,
      "grad_norm": 2.943512201309204,
      "learning_rate": 0.00019644691750543767,
      "logits/chosen": -0.2617231011390686,
      "logits/rejected": -0.38011983036994934,
      "logps/chosen": -10.003762245178223,
      "logps/rejected": -27.785303115844727,
      "loss": 1.1763017177581787,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6572035551071167,
      "rewards/accuracies": 0.671875,
      "rewards/chosen": -0.07653994858264923,
      "rewards/margins": 0.9367327690124512,
      "rewards/rejected": -1.0132726430892944,
      "step": 59,
      "train_speed(iter/s)": 0.005615
    },
    {
      "epoch": 0.2684563758389262,
      "grad_norm": 1.736883521080017,
      "learning_rate": 0.00019624804319314705,
      "logits/chosen": -0.3555205464363098,
      "logits/rejected": -0.4549139738082886,
      "logps/chosen": -7.022237300872803,
      "logps/rejected": -28.04250717163086,
      "loss": 0.9159735441207886,
      "memory(GiB)": 46.57,
      "nll_loss": 0.47989311814308167,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 0.060450125485658646,
      "rewards/margins": 1.1664522886276245,
      "rewards/rejected": -1.1060020923614502,
      "step": 60,
      "train_speed(iter/s)": 0.005616
    },
    {
      "epoch": 0.27293064876957496,
      "grad_norm": 1.8152859210968018,
      "learning_rate": 0.00019604385992733715,
      "logits/chosen": -0.2845993638038635,
      "logits/rejected": -0.3513396680355072,
      "logps/chosen": -9.077548027038574,
      "logps/rejected": -22.780668258666992,
      "loss": 0.9680572748184204,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4960174560546875,
      "rewards/accuracies": 0.734375,
      "rewards/chosen": 0.06926427036523819,
      "rewards/margins": 0.9312706589698792,
      "rewards/rejected": -0.862006425857544,
      "step": 61,
      "train_speed(iter/s)": 0.005616
    },
    {
      "epoch": 0.27740492170022374,
      "grad_norm": 2.46378755569458,
      "learning_rate": 0.00019583437897056915,
      "logits/chosen": -0.3284386098384857,
      "logits/rejected": -0.4009109139442444,
      "logps/chosen": -8.893238067626953,
      "logps/rejected": -21.27986717224121,
      "loss": 1.128482699394226,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5518131256103516,
      "rewards/accuracies": 0.609375,
      "rewards/chosen": 0.010286218486726284,
      "rewards/margins": 0.5274713635444641,
      "rewards/rejected": -0.5171850919723511,
      "step": 62,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.28187919463087246,
      "grad_norm": 2.1732046604156494,
      "learning_rate": 0.00019561961187761985,
      "logits/chosen": -0.24422433972358704,
      "logits/rejected": -0.375420480966568,
      "logps/chosen": -6.340362071990967,
      "logps/rejected": -20.184795379638672,
      "loss": 0.987078070640564,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4728015065193176,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.168831467628479,
      "rewards/margins": 0.6444362998008728,
      "rewards/rejected": -0.4756048321723938,
      "step": 63,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.28635346756152125,
      "grad_norm": 2.231933355331421,
      "learning_rate": 0.00019539957049484458,
      "logits/chosen": -0.22632239758968353,
      "logits/rejected": -0.3298073410987854,
      "logps/chosen": -5.790369987487793,
      "logps/rejected": -21.647865295410156,
      "loss": 0.9348481893539429,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4110260307788849,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.11635202169418335,
      "rewards/margins": 0.6466613411903381,
      "rewards/rejected": -0.5303093194961548,
      "step": 64,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.29082774049217003,
      "grad_norm": 2.592883825302124,
      "learning_rate": 0.00019517426695952358,
      "logits/chosen": -0.2631714344024658,
      "logits/rejected": -0.3835735321044922,
      "logps/chosen": -8.772464752197266,
      "logps/rejected": -23.13977813720703,
      "loss": 1.0820575952529907,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6241056323051453,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.1670496016740799,
      "rewards/margins": 0.9779586791992188,
      "rewards/rejected": -0.8109091520309448,
      "step": 65,
      "train_speed(iter/s)": 0.005617
    },
    {
      "epoch": 0.2953020134228188,
      "grad_norm": 2.1508402824401855,
      "learning_rate": 0.0001949437136991925,
      "logits/chosen": -0.28555068373680115,
      "logits/rejected": -0.42386704683303833,
      "logps/chosen": -4.788064002990723,
      "logps/rejected": -26.989431381225586,
      "loss": 0.7586299777030945,
      "memory(GiB)": 46.57,
      "nll_loss": 0.3784462809562683,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.09788702428340912,
      "rewards/margins": 1.3373408317565918,
      "rewards/rejected": -1.239453673362732,
      "step": 66,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.29977628635346754,
      "grad_norm": 4.065814971923828,
      "learning_rate": 0.00019470792343095718,
      "logits/chosen": -0.3336545526981354,
      "logits/rejected": -0.3990672826766968,
      "logps/chosen": -8.33244514465332,
      "logps/rejected": -29.013874053955078,
      "loss": 1.0058256387710571,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6062498092651367,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.020362887531518936,
      "rewards/margins": 1.5735464096069336,
      "rewards/rejected": -1.5939093828201294,
      "step": 67,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.3042505592841163,
      "grad_norm": 2.998826265335083,
      "learning_rate": 0.0001944669091607919,
      "logits/chosen": -0.17095105350017548,
      "logits/rejected": -0.33402499556541443,
      "logps/chosen": -7.172041893005371,
      "logps/rejected": -36.20872497558594,
      "loss": 0.8435176610946655,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5263906121253967,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.01939973793923855,
      "rewards/margins": 2.0239157676696777,
      "rewards/rejected": -2.004516124725342,
      "step": 68,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.3087248322147651,
      "grad_norm": 2.8316242694854736,
      "learning_rate": 0.00019422068418282202,
      "logits/chosen": -0.23822994530200958,
      "logits/rejected": -0.3510916531085968,
      "logps/chosen": -8.204334259033203,
      "logps/rejected": -26.101930618286133,
      "loss": 0.9929779171943665,
      "memory(GiB)": 46.57,
      "nll_loss": 0.592566967010498,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.07919630408287048,
      "rewards/margins": 1.4216609001159668,
      "rewards/rejected": -1.3424646854400635,
      "step": 69,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.3131991051454139,
      "grad_norm": 2.1727654933929443,
      "learning_rate": 0.00019396926207859084,
      "logits/chosen": -0.19429409503936768,
      "logits/rejected": -0.33817699551582336,
      "logps/chosen": -7.17151403427124,
      "logps/rejected": -29.4168701171875,
      "loss": 0.8446124196052551,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4697123169898987,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.09260577708482742,
      "rewards/margins": 1.373153567314148,
      "rewards/rejected": -1.280547857284546,
      "step": 70,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.31767337807606266,
      "grad_norm": 2.131971597671509,
      "learning_rate": 0.00019371265671631037,
      "logits/chosen": -0.2500344514846802,
      "logits/rejected": -0.391252338886261,
      "logps/chosen": -6.454396724700928,
      "logps/rejected": -31.325042724609375,
      "loss": 0.857275128364563,
      "memory(GiB)": 46.57,
      "nll_loss": 0.45489931106567383,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.12299442291259766,
      "rewards/margins": 1.4080641269683838,
      "rewards/rejected": -1.2850697040557861,
      "step": 71,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.3221476510067114,
      "grad_norm": 2.6743502616882324,
      "learning_rate": 0.00019345088225009626,
      "logits/chosen": -0.20935188233852386,
      "logits/rejected": -0.3504689633846283,
      "logps/chosen": -8.028803825378418,
      "logps/rejected": -33.473976135253906,
      "loss": 0.951294481754303,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5516812205314636,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.002318259794265032,
      "rewards/margins": 1.4767383337020874,
      "rewards/rejected": -1.4744200706481934,
      "step": 72,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.32662192393736017,
      "grad_norm": 2.1057374477386475,
      "learning_rate": 0.0001931839531191873,
      "logits/chosen": -0.21616998314857483,
      "logits/rejected": -0.3215841054916382,
      "logps/chosen": -6.386109352111816,
      "logps/rejected": -26.058643341064453,
      "loss": 0.9168461561203003,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5115207433700562,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.16049177944660187,
      "rewards/margins": 1.1623506546020508,
      "rewards/rejected": -1.0018587112426758,
      "step": 73,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.33109619686800895,
      "grad_norm": 1.8313034772872925,
      "learning_rate": 0.00019291188404714878,
      "logits/chosen": -0.2762296795845032,
      "logits/rejected": -0.3759133815765381,
      "logps/chosen": -7.998234748840332,
      "logps/rejected": -29.753068923950195,
      "loss": 0.9865212440490723,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5828378796577454,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.04827752709388733,
      "rewards/margins": 1.679243564605713,
      "rewards/rejected": -1.6309659481048584,
      "step": 74,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.33557046979865773,
      "grad_norm": 2.7458698749542236,
      "learning_rate": 0.0001926346900410604,
      "logits/chosen": -0.26945143938064575,
      "logits/rejected": -0.319683313369751,
      "logps/chosen": -9.152531623840332,
      "logps/rejected": -22.447328567504883,
      "loss": 1.118042230606079,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6387289762496948,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.012468192726373672,
      "rewards/margins": 0.904109537601471,
      "rewards/rejected": -0.8916412591934204,
      "step": 75,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.3400447427293065,
      "grad_norm": 2.5607635974884033,
      "learning_rate": 0.00019235238639068856,
      "logits/chosen": -0.19932252168655396,
      "logits/rejected": -0.37197503447532654,
      "logps/chosen": -7.51588773727417,
      "logps/rejected": -33.714637756347656,
      "loss": 0.9608560800552368,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5613290667533875,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0078031644225120544,
      "rewards/margins": 1.3550933599472046,
      "rewards/rejected": -1.347290277481079,
      "step": 76,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.34451901565995524,
      "grad_norm": 2.152766227722168,
      "learning_rate": 0.00019206498866764288,
      "logits/chosen": -0.18406139314174652,
      "logits/rejected": -0.36746495962142944,
      "logps/chosen": -4.768266677856445,
      "logps/rejected": -35.82173156738281,
      "loss": 0.8752678632736206,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5213672518730164,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": -0.024980902671813965,
      "rewards/margins": 1.6603260040283203,
      "rewards/rejected": -1.6853069067001343,
      "step": 77,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.348993288590604,
      "grad_norm": 1.957088589668274,
      "learning_rate": 0.0001917725127245174,
      "logits/chosen": -0.2368219494819641,
      "logits/rejected": -0.31324779987335205,
      "logps/chosen": -7.644411087036133,
      "logps/rejected": -25.68874740600586,
      "loss": 0.9143100380897522,
      "memory(GiB)": 46.57,
      "nll_loss": 0.47288480401039124,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.04239855334162712,
      "rewards/margins": 0.925534188747406,
      "rewards/rejected": -0.8831356167793274,
      "step": 78,
      "train_speed(iter/s)": 0.005618
    },
    {
      "epoch": 0.3534675615212528,
      "grad_norm": 2.0980629920959473,
      "learning_rate": 0.0001914749746940161,
      "logits/chosen": -0.3016536831855774,
      "logits/rejected": -0.3982566297054291,
      "logps/chosen": -7.566556930541992,
      "logps/rejected": -26.53971290588379,
      "loss": 0.9413108825683594,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5161536931991577,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": 0.1749323159456253,
      "rewards/margins": 1.1360664367675781,
      "rewards/rejected": -0.9611341953277588,
      "step": 79,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.3579418344519016,
      "grad_norm": 2.1753523349761963,
      "learning_rate": 0.00019117239098806295,
      "logits/chosen": -0.25202423334121704,
      "logits/rejected": -0.3296201527118683,
      "logps/chosen": -8.623554229736328,
      "logps/rejected": -24.96216583251953,
      "loss": 0.9156360626220703,
      "memory(GiB)": 46.57,
      "nll_loss": 0.521922767162323,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17782656848430634,
      "rewards/margins": 1.1749255657196045,
      "rewards/rejected": -0.997098982334137,
      "step": 80,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.3624161073825503,
      "grad_norm": 3.2816970348358154,
      "learning_rate": 0.00019086477829689685,
      "logits/chosen": -0.22371181845664978,
      "logits/rejected": -0.3009035885334015,
      "logps/chosen": -10.19958782196045,
      "logps/rejected": -26.129743576049805,
      "loss": 1.161747932434082,
      "memory(GiB)": 46.57,
      "nll_loss": 0.7456332445144653,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.06418374180793762,
      "rewards/margins": 1.1640924215316772,
      "rewards/rejected": -1.099908709526062,
      "step": 81,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.3668903803131991,
      "grad_norm": 4.27427339553833,
      "learning_rate": 0.0001905521535881509,
      "logits/chosen": -0.15121029317378998,
      "logits/rejected": -0.3036370873451233,
      "logps/chosen": -7.66187858581543,
      "logps/rejected": -30.927831649780273,
      "loss": 1.1210309267044067,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6641408205032349,
      "rewards/accuracies": 0.765625,
      "rewards/chosen": -0.00867693591862917,
      "rewards/margins": 1.272807002067566,
      "rewards/rejected": -1.2814840078353882,
      "step": 82,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.3713646532438479,
      "grad_norm": 2.29978084564209,
      "learning_rate": 0.00019023453410591635,
      "logits/chosen": -0.21486631035804749,
      "logits/rejected": -0.3746005892753601,
      "logps/chosen": -9.569436073303223,
      "logps/rejected": -32.21381378173828,
      "loss": 1.1229475736618042,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6794871687889099,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.0131741464138031,
      "rewards/margins": 1.4518442153930664,
      "rewards/rejected": -1.4386701583862305,
      "step": 83,
      "train_speed(iter/s)": 0.005619
    },
    {
      "epoch": 0.37583892617449666,
      "grad_norm": 2.6593234539031982,
      "learning_rate": 0.00018991193736979175,
      "logits/chosen": -0.12227392941713333,
      "logits/rejected": -0.2726154327392578,
      "logps/chosen": -7.078165054321289,
      "logps/rejected": -30.666397094726562,
      "loss": 0.7911984920501709,
      "memory(GiB)": 46.57,
      "nll_loss": 0.4572717845439911,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": 0.06052548438310623,
      "rewards/margins": 1.428847074508667,
      "rewards/rejected": -1.3683216571807861,
      "step": 84,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.38031319910514544,
      "grad_norm": 2.8958120346069336,
      "learning_rate": 0.00018958438117391618,
      "logits/chosen": -0.0972181037068367,
      "logits/rejected": -0.2590752840042114,
      "logps/chosen": -8.193891525268555,
      "logps/rejected": -35.56235885620117,
      "loss": 1.0837842226028442,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6906030774116516,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.06835082173347473,
      "rewards/margins": 1.79606294631958,
      "rewards/rejected": -1.8644137382507324,
      "step": 85,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.38478747203579416,
      "grad_norm": 3.5083513259887695,
      "learning_rate": 0.00018925188358598813,
      "logits/chosen": -0.17531399428844452,
      "logits/rejected": -0.25995591282844543,
      "logps/chosen": -8.639050483703613,
      "logps/rejected": -29.226892471313477,
      "loss": 1.0494576692581177,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6720495820045471,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.03724604845046997,
      "rewards/margins": 1.4595208168029785,
      "rewards/rejected": -1.4967668056488037,
      "step": 86,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.38926174496644295,
      "grad_norm": 2.0132222175598145,
      "learning_rate": 0.00018891446294626866,
      "logits/chosen": -0.05051232874393463,
      "logits/rejected": -0.25516456365585327,
      "logps/chosen": -4.587294578552246,
      "logps/rejected": -36.072322845458984,
      "loss": 0.602871298789978,
      "memory(GiB)": 46.57,
      "nll_loss": 0.32869917154312134,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.17169487476348877,
      "rewards/margins": 2.032069444656372,
      "rewards/rejected": -1.8603744506835938,
      "step": 87,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.39373601789709173,
      "grad_norm": 3.1251611709594727,
      "learning_rate": 0.00018857213786656985,
      "logits/chosen": -0.16258063912391663,
      "logits/rejected": -0.2599830627441406,
      "logps/chosen": -9.219761848449707,
      "logps/rejected": -27.451026916503906,
      "loss": 1.0880504846572876,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6885117292404175,
      "rewards/accuracies": 0.859375,
      "rewards/chosen": 0.04242437332868576,
      "rewards/margins": 1.207942247390747,
      "rewards/rejected": -1.1655179262161255,
      "step": 88,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.3982102908277405,
      "grad_norm": 2.0855207443237305,
      "learning_rate": 0.0001882249272292282,
      "logits/chosen": -0.1683093011379242,
      "logits/rejected": -0.3001120686531067,
      "logps/chosen": -7.336337089538574,
      "logps/rejected": -24.294296264648438,
      "loss": 0.9775981903076172,
      "memory(GiB)": 46.57,
      "nll_loss": 0.515565812587738,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.09023457765579224,
      "rewards/margins": 0.9091135859489441,
      "rewards/rejected": -0.8188790082931519,
      "step": 89,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.40268456375838924,
      "grad_norm": 1.8228232860565186,
      "learning_rate": 0.00018787285018606297,
      "logits/chosen": -0.1059221625328064,
      "logits/rejected": -0.30855828523635864,
      "logps/chosen": -3.3506364822387695,
      "logps/rejected": -32.792762756347656,
      "loss": 0.6121629476547241,
      "memory(GiB)": 46.57,
      "nll_loss": 0.2924441993236542,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.11779851466417313,
      "rewards/margins": 1.558281660079956,
      "rewards/rejected": -1.4404832124710083,
      "step": 90,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.407158836689038,
      "grad_norm": 2.038564443588257,
      "learning_rate": 0.00018751592615732005,
      "logits/chosen": -0.23549917340278625,
      "logits/rejected": -0.28893160820007324,
      "logps/chosen": -7.049911975860596,
      "logps/rejected": -23.307397842407227,
      "loss": 0.8128443956375122,
      "memory(GiB)": 46.57,
      "nll_loss": 0.42805472016334534,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.235755056142807,
      "rewards/margins": 1.322766900062561,
      "rewards/rejected": -1.0870119333267212,
      "step": 91,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.4116331096196868,
      "grad_norm": 3.0157337188720703,
      "learning_rate": 0.0001871541748306005,
      "logits/chosen": -0.16576483845710754,
      "logits/rejected": -0.3216527998447418,
      "logps/chosen": -3.5259528160095215,
      "logps/rejected": -34.49477005004883,
      "loss": 0.6575559377670288,
      "memory(GiB)": 46.57,
      "nll_loss": 0.324476957321167,
      "rewards/accuracies": 0.796875,
      "rewards/chosen": 0.0956890657544136,
      "rewards/margins": 2.1255316734313965,
      "rewards/rejected": -2.0298423767089844,
      "step": 92,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.4161073825503356,
      "grad_norm": 4.184682846069336,
      "learning_rate": 0.00018678761615977468,
      "logits/chosen": -0.08753462135791779,
      "logits/rejected": -0.2798512876033783,
      "logps/chosen": -9.157657623291016,
      "logps/rejected": -36.74687576293945,
      "loss": 1.0873180627822876,
      "memory(GiB)": 46.57,
      "nll_loss": 0.7294520139694214,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": 0.07273296266794205,
      "rewards/margins": 1.809075117111206,
      "rewards/rejected": -1.736342430114746,
      "step": 93,
      "train_speed(iter/s)": 0.00562
    },
    {
      "epoch": 0.42058165548098436,
      "grad_norm": 3.6453614234924316,
      "learning_rate": 0.00018641627036388169,
      "logits/chosen": -0.26587241888046265,
      "logits/rejected": -0.35532236099243164,
      "logps/chosen": -6.705877780914307,
      "logps/rejected": -34.05841827392578,
      "loss": 0.8682178258895874,
      "memory(GiB)": 46.57,
      "nll_loss": 0.5556175112724304,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0699356198310852,
      "rewards/margins": 2.082659959793091,
      "rewards/rejected": -2.0127243995666504,
      "step": 94,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.4250559284116331,
      "grad_norm": 3.926342725753784,
      "learning_rate": 0.00018604015792601396,
      "logits/chosen": -0.18198826909065247,
      "logits/rejected": -0.28424450755119324,
      "logps/chosen": -9.228765487670898,
      "logps/rejected": -28.494712829589844,
      "loss": 1.100342869758606,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6449709534645081,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0022647292353212833,
      "rewards/margins": 1.3801387548446655,
      "rewards/rejected": -1.3778741359710693,
      "step": 95,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.42953020134228187,
      "grad_norm": 6.2491960525512695,
      "learning_rate": 0.00018565929959218758,
      "logits/chosen": -0.162868469953537,
      "logits/rejected": -0.35664084553718567,
      "logps/chosen": -5.505702495574951,
      "logps/rejected": -40.82157516479492,
      "loss": 0.8373050689697266,
      "memory(GiB)": 46.57,
      "nll_loss": 0.48504531383514404,
      "rewards/accuracies": 0.828125,
      "rewards/chosen": 0.03489302098751068,
      "rewards/margins": 1.9277093410491943,
      "rewards/rejected": -1.892816424369812,
      "step": 96,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.43400447427293065,
      "grad_norm": 3.155294895172119,
      "learning_rate": 0.0001852737163701979,
      "logits/chosen": -0.14200377464294434,
      "logits/rejected": -0.3238810896873474,
      "logps/chosen": -6.1366353034973145,
      "logps/rejected": -36.68225860595703,
      "loss": 0.7804915904998779,
      "memory(GiB)": 46.57,
      "nll_loss": 0.47483596205711365,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.05882677063345909,
      "rewards/margins": 1.9737701416015625,
      "rewards/rejected": -1.9149433374404907,
      "step": 97,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.43847874720357943,
      "grad_norm": 2.6832268238067627,
      "learning_rate": 0.00018488342952846073,
      "logits/chosen": -0.12941551208496094,
      "logits/rejected": -0.2959509789943695,
      "logps/chosen": -12.032288551330566,
      "logps/rejected": -35.381710052490234,
      "loss": 0.838473916053772,
      "memory(GiB)": 46.57,
      "nll_loss": 0.49494707584381104,
      "rewards/accuracies": 0.875,
      "rewards/chosen": 0.032307837158441544,
      "rewards/margins": 1.5512968301773071,
      "rewards/rejected": -1.5189889669418335,
      "step": 98,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.4429530201342282,
      "grad_norm": 2.0462474822998047,
      "learning_rate": 0.0001844884605948392,
      "logits/chosen": -0.23558180034160614,
      "logits/rejected": -0.3692886531352997,
      "logps/chosen": -9.6895112991333,
      "logps/rejected": -26.33759117126465,
      "loss": 1.0117727518081665,
      "memory(GiB)": 46.57,
      "nll_loss": 0.6607507467269897,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": 0.148264542222023,
      "rewards/margins": 1.4499516487121582,
      "rewards/rejected": -1.3016871213912964,
      "step": 99,
      "train_speed(iter/s)": 0.005621
    },
    {
      "epoch": 0.44742729306487694,
      "grad_norm": 2.056586503982544,
      "learning_rate": 0.00018408883135545632,
      "logits/chosen": -0.20895381271839142,
      "logits/rejected": -0.350538432598114,
      "logps/chosen": -7.372556209564209,
      "logps/rejected": -30.752912521362305,
      "loss": 0.8857133388519287,
      "memory(GiB)": 46.57,
      "nll_loss": 0.49547722935676575,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": 0.0997147411108017,
      "rewards/margins": 1.3622983694076538,
      "rewards/rejected": -1.262583613395691,
      "step": 100,
      "train_speed(iter/s)": 0.005621
    }
  ],
  "logging_steps": 1,
  "max_steps": 446,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 1.1766177885008691e+17,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}