{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 3.2248189449310303,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.417102575302124,
      "logits/rejected": -1.3289285898208618,
      "logps/chosen": -223.09201049804688,
      "logps/rejected": -281.4899597167969,
      "loss": 0.6927,
      "rewards/accuracies": 0.33125001192092896,
      "rewards/chosen": 8.828159479890019e-05,
      "rewards/margins": 0.0009968661470338702,
      "rewards/rejected": -0.0009085845085792243,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 3.0620968341827393,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.4176433086395264,
      "logits/rejected": -1.4581596851348877,
      "logps/chosen": -227.9161834716797,
      "logps/rejected": -281.8757629394531,
      "loss": 0.6906,
      "rewards/accuracies": 0.643750011920929,
      "rewards/chosen": -0.0013590670423582196,
      "rewards/margins": 0.005104787182062864,
      "rewards/rejected": -0.006463854573667049,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 2.624357223510742,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.3286405801773071,
      "logits/rejected": -1.2039461135864258,
      "logps/chosen": -220.56112670898438,
      "logps/rejected": -255.6830596923828,
      "loss": 0.6861,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": -0.004264736548066139,
      "rewards/margins": 0.014199944213032722,
      "rewards/rejected": -0.01846468076109886,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": 2.8290464878082275,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.3126474618911743,
      "logits/rejected": -1.2001457214355469,
      "logps/chosen": -231.92227172851562,
      "logps/rejected": -240.58273315429688,
      "loss": 0.6805,
      "rewards/accuracies": 0.731249988079071,
      "rewards/chosen": -0.007818732410669327,
      "rewards/margins": 0.02588127925992012,
      "rewards/rejected": -0.03370000794529915,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 2.9549717903137207,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.4152873754501343,
      "logits/rejected": -1.34293532371521,
      "logps/chosen": -228.6177215576172,
      "logps/rejected": -274.3415222167969,
      "loss": 0.67,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.017063328996300697,
      "rewards/margins": 0.04786022752523422,
      "rewards/rejected": -0.06492356210947037,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.4597318172454834,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.397507667541504,
      "logits/rejected": -1.3085404634475708,
      "logps/chosen": -207.57083129882812,
      "logps/rejected": -246.46914672851562,
      "loss": 0.6596,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.028314024209976196,
      "rewards/margins": 0.0706639513373375,
      "rewards/rejected": -0.09897797554731369,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 2.903716802597046,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.4894464015960693,
      "logits/rejected": -1.3991295099258423,
      "logps/chosen": -218.21859741210938,
      "logps/rejected": -259.07818603515625,
      "loss": 0.6396,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.03908306732773781,
      "rewards/margins": 0.11545582115650177,
      "rewards/rejected": -0.15453888475894928,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.7245819568634033,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.4166643619537354,
      "logits/rejected": -1.2851589918136597,
      "logps/chosen": -237.89370727539062,
      "logps/rejected": -270.0038757324219,
      "loss": 0.6095,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.08339570462703705,
      "rewards/margins": 0.18969933688640594,
      "rewards/rejected": -0.273095041513443,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 3.0906941890716553,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.4081467390060425,
      "logits/rejected": -1.3213756084442139,
      "logps/chosen": -245.74490356445312,
      "logps/rejected": -272.46575927734375,
      "loss": 0.6033,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.1594085991382599,
      "rewards/margins": 0.21051104366779327,
      "rewards/rejected": -0.36991962790489197,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 2.9989840984344482,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.425335168838501,
      "logits/rejected": -1.4387575387954712,
      "logps/chosen": -219.0963592529297,
      "logps/rejected": -262.6189270019531,
      "loss": 0.5662,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.1677021086215973,
      "rewards/margins": 0.3115597367286682,
      "rewards/rejected": -0.4792618155479431,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 3.8567123413085938,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.4213941097259521,
      "logits/rejected": -1.2188364267349243,
      "logps/chosen": -282.511962890625,
      "logps/rejected": -260.6012268066406,
      "loss": 0.5587,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.2968081831932068,
      "rewards/margins": 0.33465075492858887,
      "rewards/rejected": -0.6314589381217957,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 2.7673559188842773,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.3809436559677124,
      "logits/rejected": -1.3605387210845947,
      "logps/chosen": -227.60287475585938,
      "logps/rejected": -268.99884033203125,
      "loss": 0.5575,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.32013896107673645,
      "rewards/margins": 0.38853272795677185,
      "rewards/rejected": -0.7086717486381531,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.7526016235351562,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.500636339187622,
      "logits/rejected": -1.3579481840133667,
      "logps/chosen": -251.8033447265625,
      "logps/rejected": -282.87042236328125,
      "loss": 0.4822,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.3049132227897644,
      "rewards/margins": 0.5729039907455444,
      "rewards/rejected": -0.8778173327445984,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 2.671469211578369,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.466827630996704,
      "logits/rejected": -1.3935381174087524,
      "logps/chosen": -193.24647521972656,
      "logps/rejected": -263.76239013671875,
      "loss": 0.4588,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.3088843524456024,
      "rewards/margins": 0.6813338994979858,
      "rewards/rejected": -0.9902183413505554,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 3.234593629837036,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.5930967330932617,
      "logits/rejected": -1.4225651025772095,
      "logps/chosen": -220.28488159179688,
      "logps/rejected": -262.43218994140625,
      "loss": 0.4809,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": -0.4655492901802063,
      "rewards/margins": 0.7154077291488647,
      "rewards/rejected": -1.1809570789337158,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 3.725409746170044,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.5412805080413818,
      "logits/rejected": -1.3626563549041748,
      "logps/chosen": -234.2522735595703,
      "logps/rejected": -259.58172607421875,
      "loss": 0.4561,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.5094849467277527,
      "rewards/margins": 0.8051780462265015,
      "rewards/rejected": -1.3146629333496094,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 3.086479902267456,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.6061999797821045,
      "logits/rejected": -1.5315011739730835,
      "logps/chosen": -214.01416015625,
      "logps/rejected": -277.1407165527344,
      "loss": 0.3968,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.3647157847881317,
      "rewards/margins": 1.1234705448150635,
      "rewards/rejected": -1.488186240196228,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.2907378673553467,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.617488145828247,
      "logits/rejected": -1.4285557270050049,
      "logps/chosen": -215.92794799804688,
      "logps/rejected": -293.9332580566406,
      "loss": 0.3555,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.4863402247428894,
      "rewards/margins": 1.2553744316101074,
      "rewards/rejected": -1.7417147159576416,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 2.879300117492676,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.4808334112167358,
      "logits/rejected": -1.4129270315170288,
      "logps/chosen": -224.61282348632812,
      "logps/rejected": -287.599609375,
      "loss": 0.3854,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.5822515487670898,
      "rewards/margins": 1.2435760498046875,
      "rewards/rejected": -1.8258273601531982,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 2.7722413539886475,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.5995476245880127,
      "logits/rejected": -1.5469409227371216,
      "logps/chosen": -218.4732666015625,
      "logps/rejected": -274.4031982421875,
      "loss": 0.3723,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.623435914516449,
      "rewards/margins": 1.2485257387161255,
      "rewards/rejected": -1.8719615936279297,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 2.1027016639709473,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.5575580596923828,
      "logits/rejected": -1.418176293373108,
      "logps/chosen": -238.68490600585938,
      "logps/rejected": -300.532470703125,
      "loss": 0.3361,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5908123254776001,
      "rewards/margins": 1.523303747177124,
      "rewards/rejected": -2.1141159534454346,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 3.443408727645874,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.5500075817108154,
      "logits/rejected": -1.5055526494979858,
      "logps/chosen": -234.66940307617188,
      "logps/rejected": -267.90087890625,
      "loss": 0.4005,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.7276199460029602,
      "rewards/margins": 1.2876373529434204,
      "rewards/rejected": -2.0152573585510254,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 3.1174283027648926,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.5923100709915161,
      "logits/rejected": -1.4889472723007202,
      "logps/chosen": -225.01779174804688,
      "logps/rejected": -288.71173095703125,
      "loss": 0.3272,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6660363078117371,
      "rewards/margins": 1.6434768438339233,
      "rewards/rejected": -2.3095130920410156,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 3.5097219944000244,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.5631916522979736,
      "logits/rejected": -1.5810332298278809,
      "logps/chosen": -216.34115600585938,
      "logps/rejected": -309.64642333984375,
      "loss": 0.2956,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.607854962348938,
      "rewards/margins": 1.7155084609985352,
      "rewards/rejected": -2.3233633041381836,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 3.2788164615631104,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.5768873691558838,
      "logits/rejected": -1.4464285373687744,
      "logps/chosen": -227.64956665039062,
      "logps/rejected": -267.4158935546875,
      "loss": 0.3828,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6830345392227173,
      "rewards/margins": 1.5734184980392456,
      "rewards/rejected": -2.256453037261963,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 2.1826467514038086,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.6085306406021118,
      "logits/rejected": -1.5910682678222656,
      "logps/chosen": -232.57266235351562,
      "logps/rejected": -304.3570556640625,
      "loss": 0.294,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5465613603591919,
      "rewards/margins": 1.9157283306121826,
      "rewards/rejected": -2.462289571762085,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 2.313305377960205,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.6053348779678345,
      "logits/rejected": -1.5949219465255737,
      "logps/chosen": -215.5905303955078,
      "logps/rejected": -289.11212158203125,
      "loss": 0.2507,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.41833844780921936,
      "rewards/margins": 2.0826306343078613,
      "rewards/rejected": -2.500969409942627,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 3.754728078842163,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.5684372186660767,
      "logits/rejected": -1.4641824960708618,
      "logps/chosen": -210.9490203857422,
      "logps/rejected": -281.46337890625,
      "loss": 0.3506,
      "rewards/accuracies": 0.831250011920929,
      "rewards/chosen": -0.6244282722473145,
      "rewards/margins": 1.8182014226913452,
      "rewards/rejected": -2.442629814147949,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 4.167830467224121,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.5899982452392578,
      "logits/rejected": -1.486648678779602,
      "logps/chosen": -225.3086700439453,
      "logps/rejected": -274.19500732421875,
      "loss": 0.3288,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.6446894407272339,
      "rewards/margins": 1.9515396356582642,
      "rewards/rejected": -2.596228837966919,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 4.168672561645508,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.5442506074905396,
      "logits/rejected": -1.4385302066802979,
      "logps/chosen": -218.75851440429688,
      "logps/rejected": -283.3717041015625,
      "loss": 0.3998,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": -0.570462703704834,
      "rewards/margins": 1.811073899269104,
      "rewards/rejected": -2.3815362453460693,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 2.5970113277435303,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.6545788049697876,
      "logits/rejected": -1.5757300853729248,
      "logps/chosen": -226.00436401367188,
      "logps/rejected": -298.34307861328125,
      "loss": 0.3149,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.5687687397003174,
      "rewards/margins": 2.034900426864624,
      "rewards/rejected": -2.6036694049835205,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 2.196605682373047,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.725408911705017,
      "logits/rejected": -1.5622869729995728,
      "logps/chosen": -224.3115692138672,
      "logps/rejected": -299.6641540527344,
      "loss": 0.2456,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4611068367958069,
      "rewards/margins": 2.2169716358184814,
      "rewards/rejected": -2.6780784130096436,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.2756359577178955,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.5784146785736084,
      "logits/rejected": -1.459627628326416,
      "logps/chosen": -237.4366912841797,
      "logps/rejected": -282.8692321777344,
      "loss": 0.3194,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.42128247022628784,
      "rewards/margins": 1.9330406188964844,
      "rewards/rejected": -2.354323148727417,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 2.7691073417663574,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.6254955530166626,
      "logits/rejected": -1.5783421993255615,
      "logps/chosen": -210.7351837158203,
      "logps/rejected": -305.00836181640625,
      "loss": 0.2511,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.3136264681816101,
      "rewards/margins": 2.3627917766571045,
      "rewards/rejected": -2.6764187812805176,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 2.728076934814453,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.6129728555679321,
      "logits/rejected": -1.4741400480270386,
      "logps/chosen": -242.5968475341797,
      "logps/rejected": -298.1058654785156,
      "loss": 0.2798,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.3989323079586029,
      "rewards/margins": 2.153937816619873,
      "rewards/rejected": -2.552870035171509,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 3.1587653160095215,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.6577237844467163,
      "logits/rejected": -1.5179884433746338,
      "logps/chosen": -258.1352233886719,
      "logps/rejected": -287.0603942871094,
      "loss": 0.3108,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.6214097142219543,
      "rewards/margins": 2.040191650390625,
      "rewards/rejected": -2.6616015434265137,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 3.1338894367218018,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.5607925653457642,
      "logits/rejected": -1.4988266229629517,
      "logps/chosen": -211.45278930664062,
      "logps/rejected": -289.1008605957031,
      "loss": 0.2638,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.3999144434928894,
      "rewards/margins": 2.4039969444274902,
      "rewards/rejected": -2.8039114475250244,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 2.9660542011260986,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.680594801902771,
      "logits/rejected": -1.4900007247924805,
      "logps/chosen": -219.1302947998047,
      "logps/rejected": -285.2054443359375,
      "loss": 0.2477,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.34523534774780273,
      "rewards/margins": 2.4674582481384277,
      "rewards/rejected": -2.8126938343048096,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 3.570490598678589,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.6982581615447998,
      "logits/rejected": -1.5957467555999756,
      "logps/chosen": -224.43197631835938,
      "logps/rejected": -306.5550842285156,
      "loss": 0.2465,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.5012459754943848,
      "rewards/margins": 2.4313809871673584,
      "rewards/rejected": -2.932626724243164,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 2.77095627784729,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.5608937740325928,
      "logits/rejected": -1.448979377746582,
      "logps/chosen": -285.95556640625,
      "logps/rejected": -297.87982177734375,
      "loss": 0.2754,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.3715810179710388,
      "rewards/margins": 2.369892120361328,
      "rewards/rejected": -2.7414729595184326,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 2.7541775703430176,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.6883554458618164,
      "logits/rejected": -1.579251766204834,
      "logps/chosen": -193.3550567626953,
      "logps/rejected": -294.20953369140625,
      "loss": 0.2067,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4765893518924713,
      "rewards/margins": 2.9404797554016113,
      "rewards/rejected": -3.4170689582824707,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 3.730454206466675,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.7941776514053345,
      "logits/rejected": -1.643109679222107,
      "logps/chosen": -230.3562469482422,
      "logps/rejected": -287.34381103515625,
      "loss": 0.2412,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6958181858062744,
      "rewards/margins": 2.7433438301086426,
      "rewards/rejected": -3.439162015914917,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 2.533442497253418,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.6759326457977295,
      "logits/rejected": -1.5923807621002197,
      "logps/chosen": -238.96255493164062,
      "logps/rejected": -312.38726806640625,
      "loss": 0.2548,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.8674060702323914,
      "rewards/margins": 2.7441043853759766,
      "rewards/rejected": -3.6115105152130127,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 2.9548909664154053,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.6703212261199951,
      "logits/rejected": -1.5990310907363892,
      "logps/chosen": -249.1092071533203,
      "logps/rejected": -305.9916687011719,
      "loss": 0.2558,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7586711645126343,
      "rewards/margins": 2.6208558082580566,
      "rewards/rejected": -3.3795273303985596,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 3.1150755882263184,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.6585559844970703,
      "logits/rejected": -1.5935546159744263,
      "logps/chosen": -226.49783325195312,
      "logps/rejected": -299.1880187988281,
      "loss": 0.2518,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6116968393325806,
      "rewards/margins": 2.668179750442505,
      "rewards/rejected": -3.279876232147217,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 2.324882745742798,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.648912787437439,
      "logits/rejected": -1.658570647239685,
      "logps/chosen": -215.8841094970703,
      "logps/rejected": -304.59271240234375,
      "loss": 0.2392,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.4058478772640228,
      "rewards/margins": 2.8351776599884033,
      "rewards/rejected": -3.241025447845459,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 3.683439016342163,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.8577114343643188,
      "logits/rejected": -1.7553380727767944,
      "logps/chosen": -217.13571166992188,
      "logps/rejected": -311.47308349609375,
      "loss": 0.2889,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.7178010940551758,
      "rewards/margins": 2.7824273109436035,
      "rewards/rejected": -3.5002281665802,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 2.6882176399230957,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.641305685043335,
      "logits/rejected": -1.5148091316223145,
      "logps/chosen": -220.66104125976562,
      "logps/rejected": -293.9461669921875,
      "loss": 0.2823,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.7442353963851929,
      "rewards/margins": 2.766519546508789,
      "rewards/rejected": -3.5107548236846924,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 2.4825658798217773,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.8355581760406494,
      "logits/rejected": -1.5800683498382568,
      "logps/chosen": -246.59066772460938,
      "logps/rejected": -295.7894287109375,
      "loss": 0.2453,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.5356179475784302,
      "rewards/margins": 2.7746822834014893,
      "rewards/rejected": -3.31030011177063,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 3.213561773300171,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.7159204483032227,
      "logits/rejected": -1.582465410232544,
      "logps/chosen": -226.40658569335938,
      "logps/rejected": -287.4487609863281,
      "loss": 0.251,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7098070383071899,
      "rewards/margins": 2.7901222705841064,
      "rewards/rejected": -3.499929428100586,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 2.457315444946289,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.7356235980987549,
      "logits/rejected": -1.568500280380249,
      "logps/chosen": -234.6002655029297,
      "logps/rejected": -301.8827209472656,
      "loss": 0.2264,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.7143124938011169,
      "rewards/margins": 2.9733080863952637,
      "rewards/rejected": -3.6876206398010254,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 2.9479100704193115,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.7711416482925415,
      "logits/rejected": -1.6228008270263672,
      "logps/chosen": -216.1881561279297,
      "logps/rejected": -294.84771728515625,
      "loss": 0.2206,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.7808042764663696,
      "rewards/margins": 3.0704331398010254,
      "rewards/rejected": -3.8512375354766846,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 2.788898468017578,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.7666418552398682,
      "logits/rejected": -1.5968812704086304,
      "logps/chosen": -238.59384155273438,
      "logps/rejected": -305.6246643066406,
      "loss": 0.2035,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8097629547119141,
      "rewards/margins": 3.0955402851104736,
      "rewards/rejected": -3.9053032398223877,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 3.132816791534424,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.7248319387435913,
      "logits/rejected": -1.5983197689056396,
      "logps/chosen": -229.5932159423828,
      "logps/rejected": -309.0335388183594,
      "loss": 0.2052,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.7570894956588745,
      "rewards/margins": 3.117053747177124,
      "rewards/rejected": -3.874143600463867,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 4.418814182281494,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.6472892761230469,
      "logits/rejected": -1.5750362873077393,
      "logps/chosen": -235.33743286132812,
      "logps/rejected": -306.19622802734375,
      "loss": 0.2581,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7988941669464111,
      "rewards/margins": 3.0710582733154297,
      "rewards/rejected": -3.869952440261841,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 3.9372878074645996,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.8032338619232178,
      "logits/rejected": -1.6216585636138916,
      "logps/chosen": -225.59542846679688,
      "logps/rejected": -291.0912780761719,
      "loss": 0.2022,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7199317216873169,
      "rewards/margins": 3.431600570678711,
      "rewards/rejected": -4.151532173156738,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 3.708203077316284,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.7838048934936523,
      "logits/rejected": -1.6193605661392212,
      "logps/chosen": -247.69082641601562,
      "logps/rejected": -310.2096862792969,
      "loss": 0.2188,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9647334814071655,
      "rewards/margins": 3.321147918701172,
      "rewards/rejected": -4.285881519317627,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 1.9725751876831055,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.813089370727539,
      "logits/rejected": -1.6678119897842407,
      "logps/chosen": -224.18002319335938,
      "logps/rejected": -306.02935791015625,
      "loss": 0.2495,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1165226697921753,
      "rewards/margins": 3.2529945373535156,
      "rewards/rejected": -4.369516849517822,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 5.2026777267456055,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.7238123416900635,
      "logits/rejected": -1.6830676794052124,
      "logps/chosen": -245.8369598388672,
      "logps/rejected": -311.4858093261719,
      "loss": 0.2428,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.6427221894264221,
      "rewards/margins": 3.463367462158203,
      "rewards/rejected": -4.1060895919799805,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 2.9667296409606934,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.8424475193023682,
      "logits/rejected": -1.6727721691131592,
      "logps/chosen": -232.18850708007812,
      "logps/rejected": -304.9006042480469,
      "loss": 0.199,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9507037401199341,
      "rewards/margins": 3.463369846343994,
      "rewards/rejected": -4.4140729904174805,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 2.3393447399139404,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.702309012413025,
      "logits/rejected": -1.6070747375488281,
      "logps/chosen": -245.9197540283203,
      "logps/rejected": -308.6512756347656,
      "loss": 0.2258,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.8237674832344055,
      "rewards/margins": 3.2607333660125732,
      "rewards/rejected": -4.084500789642334,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 4.266489505767822,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.7092502117156982,
      "logits/rejected": -1.588419795036316,
      "logps/chosen": -240.1862030029297,
      "logps/rejected": -300.18084716796875,
      "loss": 0.2137,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6268893480300903,
      "rewards/margins": 3.2068610191345215,
      "rewards/rejected": -3.8337504863739014,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 2.109783172607422,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.8387340307235718,
      "logits/rejected": -1.728690505027771,
      "logps/chosen": -226.1668243408203,
      "logps/rejected": -312.61566162109375,
      "loss": 0.2581,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.7394665479660034,
      "rewards/margins": 3.5360705852508545,
      "rewards/rejected": -4.27553653717041,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 2.6758790016174316,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.7539641857147217,
      "logits/rejected": -1.6158876419067383,
      "logps/chosen": -239.16726684570312,
      "logps/rejected": -306.02581787109375,
      "loss": 0.2369,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.7840451002120972,
      "rewards/margins": 3.146629810333252,
      "rewards/rejected": -3.9306750297546387,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 1.7696614265441895,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.5977143049240112,
      "logits/rejected": -1.527240514755249,
      "logps/chosen": -252.3458709716797,
      "logps/rejected": -297.1427307128906,
      "loss": 0.2069,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.7750762701034546,
      "rewards/margins": 3.285799741744995,
      "rewards/rejected": -4.06087589263916,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.7009758949279785,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.8222010135650635,
      "logits/rejected": -1.6697711944580078,
      "logps/chosen": -236.01950073242188,
      "logps/rejected": -309.9307861328125,
      "loss": 0.2076,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6853605508804321,
      "rewards/margins": 3.5107035636901855,
      "rewards/rejected": -4.196064472198486,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 2.219874143600464,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.7028766870498657,
      "logits/rejected": -1.6242986917495728,
      "logps/chosen": -228.4136505126953,
      "logps/rejected": -323.2743835449219,
      "loss": 0.1681,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.4335177540779114,
      "rewards/margins": 3.8759121894836426,
      "rewards/rejected": -4.309430122375488,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 3.1199467182159424,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.7407314777374268,
      "logits/rejected": -1.668487548828125,
      "logps/chosen": -231.2909698486328,
      "logps/rejected": -320.69512939453125,
      "loss": 0.1919,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6203645467758179,
      "rewards/margins": 3.9102706909179688,
      "rewards/rejected": -4.530634880065918,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 1.6120609045028687,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.7064342498779297,
      "logits/rejected": -1.6468336582183838,
      "logps/chosen": -221.8558349609375,
      "logps/rejected": -309.10345458984375,
      "loss": 0.1907,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.9249537587165833,
      "rewards/margins": 3.7362067699432373,
      "rewards/rejected": -4.661160945892334,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 3.919717788696289,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.7625980377197266,
      "logits/rejected": -1.5392252206802368,
      "logps/chosen": -247.5630645751953,
      "logps/rejected": -319.32684326171875,
      "loss": 0.197,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0334270000457764,
      "rewards/margins": 4.253495216369629,
      "rewards/rejected": -5.286921977996826,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 3.917741537094116,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.8463985919952393,
      "logits/rejected": -1.6996173858642578,
      "logps/chosen": -250.5867156982422,
      "logps/rejected": -340.89422607421875,
      "loss": 0.2088,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.037157654762268,
      "rewards/margins": 4.255171298980713,
      "rewards/rejected": -5.292328834533691,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 3.468600273132324,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.6923949718475342,
      "logits/rejected": -1.5864944458007812,
      "logps/chosen": -227.09860229492188,
      "logps/rejected": -320.20343017578125,
      "loss": 0.203,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0783360004425049,
      "rewards/margins": 3.753375291824341,
      "rewards/rejected": -4.831711769104004,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 3.8932225704193115,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.6695184707641602,
      "logits/rejected": -1.5590105056762695,
      "logps/chosen": -247.1300048828125,
      "logps/rejected": -310.8145446777344,
      "loss": 0.1774,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.9608127474784851,
      "rewards/margins": 3.990257978439331,
      "rewards/rejected": -4.951070785522461,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 3.3788342475891113,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.618805170059204,
      "logits/rejected": -1.5945888757705688,
      "logps/chosen": -225.7582244873047,
      "logps/rejected": -319.0153503417969,
      "loss": 0.1866,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.9858270883560181,
      "rewards/margins": 3.8777613639831543,
      "rewards/rejected": -4.863588809967041,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 2.828134059906006,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.7046676874160767,
      "logits/rejected": -1.5746122598648071,
      "logps/chosen": -300.2704162597656,
      "logps/rejected": -307.11297607421875,
      "loss": 0.1834,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.043919324874878,
      "rewards/margins": 3.8452529907226562,
      "rewards/rejected": -4.889172554016113,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 2.781174659729004,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.7193530797958374,
      "logits/rejected": -1.5741509199142456,
      "logps/chosen": -222.92428588867188,
      "logps/rejected": -311.06689453125,
      "loss": 0.1894,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.1129997968673706,
      "rewards/margins": 3.535658597946167,
      "rewards/rejected": -4.648658275604248,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 2.395315170288086,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.6871780157089233,
      "logits/rejected": -1.5884290933609009,
      "logps/chosen": -221.77444458007812,
      "logps/rejected": -321.648681640625,
      "loss": 0.1822,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.9971541166305542,
      "rewards/margins": 4.199106216430664,
      "rewards/rejected": -5.196260452270508,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 3.117591142654419,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.5949792861938477,
      "logits/rejected": -1.5942808389663696,
      "logps/chosen": -230.6989288330078,
      "logps/rejected": -321.4884033203125,
      "loss": 0.205,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -1.0500239133834839,
      "rewards/margins": 3.972008466720581,
      "rewards/rejected": -5.022032737731934,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 2.8519601821899414,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.7170889377593994,
      "logits/rejected": -1.6525356769561768,
      "logps/chosen": -238.529296875,
      "logps/rejected": -329.4853820800781,
      "loss": 0.2118,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.1062920093536377,
      "rewards/margins": 3.961061954498291,
      "rewards/rejected": -5.06735372543335,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 1.4091684818267822,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.681089162826538,
      "logits/rejected": -1.5169637203216553,
      "logps/chosen": -227.4136199951172,
      "logps/rejected": -329.40985107421875,
      "loss": 0.1699,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3499810695648193,
      "rewards/margins": 4.325152397155762,
      "rewards/rejected": -5.67513370513916,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 2.204573392868042,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.7952913045883179,
      "logits/rejected": -1.6552798748016357,
      "logps/chosen": -227.94210815429688,
      "logps/rejected": -321.5538635253906,
      "loss": 0.1467,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.0351591110229492,
      "rewards/margins": 4.314890384674072,
      "rewards/rejected": -5.350049018859863,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 4.7300639152526855,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.7709486484527588,
      "logits/rejected": -1.5174936056137085,
      "logps/chosen": -234.28475952148438,
      "logps/rejected": -306.0198059082031,
      "loss": 0.2281,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.6678578853607178,
      "rewards/margins": 3.790971040725708,
      "rewards/rejected": -5.458829402923584,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 2.627563953399658,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.6847022771835327,
      "logits/rejected": -1.5950194597244263,
      "logps/chosen": -238.90029907226562,
      "logps/rejected": -310.26214599609375,
      "loss": 0.1812,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.1100462675094604,
      "rewards/margins": 4.378572463989258,
      "rewards/rejected": -5.488618850708008,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 3.2908780574798584,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.6498327255249023,
      "logits/rejected": -1.6628739833831787,
      "logps/chosen": -232.87203979492188,
      "logps/rejected": -300.0521240234375,
      "loss": 0.2159,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -1.1272834539413452,
      "rewards/margins": 3.792628765106201,
      "rewards/rejected": -4.919912815093994,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 1.855165958404541,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.6586782932281494,
      "logits/rejected": -1.517372965812683,
      "logps/chosen": -258.5635986328125,
      "logps/rejected": -344.59893798828125,
      "loss": 0.1408,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5040361881256104,
      "rewards/margins": 4.680717468261719,
      "rewards/rejected": -6.184753894805908,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 5.024399757385254,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.7532565593719482,
      "logits/rejected": -1.5877439975738525,
      "logps/chosen": -236.11898803710938,
      "logps/rejected": -334.73052978515625,
      "loss": 0.2139,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.5054124593734741,
      "rewards/margins": 4.465746879577637,
      "rewards/rejected": -5.971159934997559,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 2.292858362197876,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.8473304510116577,
      "logits/rejected": -1.7024672031402588,
      "logps/chosen": -237.39791870117188,
      "logps/rejected": -326.8873291015625,
      "loss": 0.2034,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.3960918188095093,
      "rewards/margins": 4.522124290466309,
      "rewards/rejected": -5.918216705322266,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 3.1267998218536377,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.7679741382598877,
      "logits/rejected": -1.6751598119735718,
      "logps/chosen": -217.69375610351562,
      "logps/rejected": -321.40911865234375,
      "loss": 0.2045,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.2226085662841797,
      "rewards/margins": 4.901395320892334,
      "rewards/rejected": -6.1240034103393555,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 2.8430871963500977,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.7301061153411865,
      "logits/rejected": -1.5988894701004028,
      "logps/chosen": -218.8601837158203,
      "logps/rejected": -315.28485107421875,
      "loss": 0.21,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -1.3717520236968994,
      "rewards/margins": 4.2111992835998535,
      "rewards/rejected": -5.582951545715332,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 2.1232240200042725,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.818627119064331,
      "logits/rejected": -1.711194396018982,
      "logps/chosen": -238.9432373046875,
      "logps/rejected": -327.90936279296875,
      "loss": 0.17,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.2995021343231201,
      "rewards/margins": 4.501131534576416,
      "rewards/rejected": -5.800633430480957,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 4.588263511657715,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.737735390663147,
      "logits/rejected": -1.6649131774902344,
      "logps/chosen": -250.17367553710938,
      "logps/rejected": -330.4911193847656,
      "loss": 0.2198,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.7452892065048218,
      "rewards/margins": 4.413934230804443,
      "rewards/rejected": -6.159224033355713,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 2.611341714859009,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.69869065284729,
      "logits/rejected": -1.5674892663955688,
      "logps/chosen": -231.4336700439453,
      "logps/rejected": -311.5548400878906,
      "loss": 0.192,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -1.5492182970046997,
      "rewards/margins": 4.648384094238281,
      "rewards/rejected": -6.19760274887085,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 4.72195291519165,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.8486287593841553,
      "logits/rejected": -1.6863996982574463,
      "logps/chosen": -240.7964324951172,
      "logps/rejected": -322.6090393066406,
      "loss": 0.1649,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.4922518730163574,
      "rewards/margins": 4.469428539276123,
      "rewards/rejected": -5.961680889129639,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 4.31331205368042,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.733007550239563,
      "logits/rejected": -1.6687922477722168,
      "logps/chosen": -235.17596435546875,
      "logps/rejected": -336.431884765625,
      "loss": 0.1628,
      "rewards/accuracies": 0.9276315569877625,
      "rewards/chosen": -1.2398635149002075,
      "rewards/margins": 4.856743812561035,
      "rewards/rejected": -6.096607208251953,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 0.9625500440597534,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.7513107061386108,
      "logits/rejected": -1.7320438623428345,
      "logps/chosen": -236.9391326904297,
      "logps/rejected": -353.11468505859375,
      "loss": 0.119,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.2717801332473755,
      "rewards/margins": 5.247875690460205,
      "rewards/rejected": -6.519656181335449,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 1.7186386585235596,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.6451976299285889,
      "logits/rejected": -1.5232036113739014,
      "logps/chosen": -248.01620483398438,
      "logps/rejected": -315.8173522949219,
      "loss": 0.1408,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2212826013565063,
      "rewards/margins": 4.718570232391357,
      "rewards/rejected": -5.939852714538574,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 2.955115795135498,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.7891451120376587,
      "logits/rejected": -1.6652504205703735,
      "logps/chosen": -222.29953002929688,
      "logps/rejected": -318.55938720703125,
      "loss": 0.1148,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5009971857070923,
      "rewards/margins": 5.109112739562988,
      "rewards/rejected": -6.610108852386475,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 1.8832951784133911,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.7436507940292358,
      "logits/rejected": -1.6489429473876953,
      "logps/chosen": -251.84146118164062,
      "logps/rejected": -335.59271240234375,
      "loss": 0.1024,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.1944636106491089,
      "rewards/margins": 5.200083255767822,
      "rewards/rejected": -6.3945465087890625,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 3.0533058643341064,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.7651679515838623,
      "logits/rejected": -1.6751089096069336,
      "logps/chosen": -241.49221801757812,
      "logps/rejected": -335.492431640625,
      "loss": 0.123,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5157369375228882,
      "rewards/margins": 5.0457444190979,
      "rewards/rejected": -6.561480522155762,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 2.936387777328491,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.6875702142715454,
      "logits/rejected": -1.6360981464385986,
      "logps/chosen": -230.61770629882812,
      "logps/rejected": -336.7130432128906,
      "loss": 0.1503,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8132297992706299,
      "rewards/margins": 5.069985866546631,
      "rewards/rejected": -6.883215427398682,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 3.192586660385132,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.817999243736267,
      "logits/rejected": -1.712794542312622,
      "logps/chosen": -217.67965698242188,
      "logps/rejected": -315.28875732421875,
      "loss": 0.1817,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.406536340713501,
      "rewards/margins": 5.129839897155762,
      "rewards/rejected": -6.536375999450684,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 4.433254718780518,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.7239211797714233,
      "logits/rejected": -1.6257041692733765,
      "logps/chosen": -254.9601287841797,
      "logps/rejected": -342.13629150390625,
      "loss": 0.1264,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.459402322769165,
      "rewards/margins": 5.333830833435059,
      "rewards/rejected": -6.793233394622803,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 3.8896450996398926,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.6991560459136963,
      "logits/rejected": -1.5692527294158936,
      "logps/chosen": -231.50454711914062,
      "logps/rejected": -332.5694885253906,
      "loss": 0.117,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4644403457641602,
      "rewards/margins": 5.550719261169434,
      "rewards/rejected": -7.015158653259277,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 2.798954963684082,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.740290641784668,
      "logits/rejected": -1.6251468658447266,
      "logps/chosen": -247.70217895507812,
      "logps/rejected": -324.9705810546875,
      "loss": 0.1123,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6576683521270752,
      "rewards/margins": 5.431394100189209,
      "rewards/rejected": -7.089062690734863,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 3.2726736068725586,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.7375444173812866,
      "logits/rejected": -1.7175487279891968,
      "logps/chosen": -254.1574249267578,
      "logps/rejected": -331.61798095703125,
      "loss": 0.1336,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.0355722904205322,
      "rewards/margins": 4.999141216278076,
      "rewards/rejected": -7.0347137451171875,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 3.4253458976745605,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.838283896446228,
      "logits/rejected": -1.6944366693496704,
      "logps/chosen": -230.379150390625,
      "logps/rejected": -352.56005859375,
      "loss": 0.1093,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7831109762191772,
      "rewards/margins": 5.638839244842529,
      "rewards/rejected": -7.421949863433838,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 1.8465055227279663,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.765180230140686,
      "logits/rejected": -1.7436046600341797,
      "logps/chosen": -231.00894165039062,
      "logps/rejected": -354.35552978515625,
      "loss": 0.1105,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.084205150604248,
      "rewards/margins": 6.009603500366211,
      "rewards/rejected": -8.093809127807617,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 2.5956075191497803,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.775918960571289,
      "logits/rejected": -1.6911274194717407,
      "logps/chosen": -240.2334747314453,
      "logps/rejected": -335.20184326171875,
      "loss": 0.1565,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8987281322479248,
      "rewards/margins": 5.527867794036865,
      "rewards/rejected": -7.426596164703369,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 2.830385684967041,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.6608412265777588,
      "logits/rejected": -1.5417561531066895,
      "logps/chosen": -261.1434326171875,
      "logps/rejected": -343.75775146484375,
      "loss": 0.1444,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7868309020996094,
      "rewards/margins": 5.3465094566345215,
      "rewards/rejected": -7.133339881896973,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 1.6458228826522827,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.682835340499878,
      "logits/rejected": -1.682253122329712,
      "logps/chosen": -228.0749969482422,
      "logps/rejected": -350.91064453125,
      "loss": 0.1117,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8750747442245483,
      "rewards/margins": 5.591343879699707,
      "rewards/rejected": -7.466418266296387,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 4.969726085662842,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.7055584192276,
      "logits/rejected": -1.6184914112091064,
      "logps/chosen": -246.9654998779297,
      "logps/rejected": -335.02557373046875,
      "loss": 0.1431,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.870309829711914,
      "rewards/margins": 5.379855155944824,
      "rewards/rejected": -7.250164031982422,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 4.240958213806152,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.6397478580474854,
      "logits/rejected": -1.5480825901031494,
      "logps/chosen": -238.30917358398438,
      "logps/rejected": -341.9546813964844,
      "loss": 0.1158,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5620613098144531,
      "rewards/margins": 5.2377424240112305,
      "rewards/rejected": -6.799803256988525,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 3.5481252670288086,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.758758544921875,
      "logits/rejected": -1.6746900081634521,
      "logps/chosen": -235.3990478515625,
      "logps/rejected": -341.65557861328125,
      "loss": 0.1671,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.0450544357299805,
      "rewards/margins": 5.749896049499512,
      "rewards/rejected": -7.794950008392334,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 2.327967882156372,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.6582815647125244,
      "logits/rejected": -1.707456350326538,
      "logps/chosen": -242.3777313232422,
      "logps/rejected": -363.1072998046875,
      "loss": 0.1118,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.587329387664795,
      "rewards/margins": 5.6046342849731445,
      "rewards/rejected": -8.191963195800781,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 1.739214539527893,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.6814908981323242,
      "logits/rejected": -1.575382947921753,
      "logps/chosen": -249.67098999023438,
      "logps/rejected": -339.2182922363281,
      "loss": 0.1331,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.7393226623535156,
      "rewards/margins": 5.339315414428711,
      "rewards/rejected": -7.078637599945068,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 5.8394670486450195,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.6644290685653687,
      "logits/rejected": -1.6048500537872314,
      "logps/chosen": -228.67880249023438,
      "logps/rejected": -353.0116271972656,
      "loss": 0.1261,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9887882471084595,
      "rewards/margins": 5.781205177307129,
      "rewards/rejected": -7.769992828369141,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 4.61240816116333,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.7240492105484009,
      "logits/rejected": -1.7118297815322876,
      "logps/chosen": -257.6966247558594,
      "logps/rejected": -348.775146484375,
      "loss": 0.1438,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.5069431066513062,
      "rewards/margins": 5.710946559906006,
      "rewards/rejected": -7.217889308929443,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 1.9813047647476196,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.636423110961914,
      "logits/rejected": -1.626508355140686,
      "logps/chosen": -241.73611450195312,
      "logps/rejected": -322.9097595214844,
      "loss": 0.1514,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.60867178440094,
      "rewards/margins": 5.148347854614258,
      "rewards/rejected": -6.757019996643066,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 5.205898284912109,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.7877315282821655,
      "logits/rejected": -1.6699727773666382,
      "logps/chosen": -235.1413116455078,
      "logps/rejected": -327.6995544433594,
      "loss": 0.2184,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -1.8307580947875977,
      "rewards/margins": 4.723857879638672,
      "rewards/rejected": -6.554616451263428,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 1.4653977155685425,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.7254440784454346,
      "logits/rejected": -1.571353554725647,
      "logps/chosen": -249.1863250732422,
      "logps/rejected": -340.4194641113281,
      "loss": 0.1343,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.406814694404602,
      "rewards/margins": 5.116804122924805,
      "rewards/rejected": -6.523619174957275,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 1.798576831817627,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.7054555416107178,
      "logits/rejected": -1.6197830438613892,
      "logps/chosen": -259.9774475097656,
      "logps/rejected": -341.87542724609375,
      "loss": 0.0874,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.6282418966293335,
      "rewards/margins": 5.709589958190918,
      "rewards/rejected": -7.337832450866699,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 5.687100887298584,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.7209688425064087,
      "logits/rejected": -1.6104001998901367,
      "logps/chosen": -253.5648956298828,
      "logps/rejected": -316.03363037109375,
      "loss": 0.1435,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.4639036655426025,
      "rewards/margins": 5.040863037109375,
      "rewards/rejected": -6.504767417907715,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 3.0057663917541504,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.7133491039276123,
      "logits/rejected": -1.6010112762451172,
      "logps/chosen": -265.01580810546875,
      "logps/rejected": -333.35260009765625,
      "loss": 0.1287,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5199475288391113,
      "rewards/margins": 5.0299577713012695,
      "rewards/rejected": -6.549904823303223,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 2.881352663040161,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.6966040134429932,
      "logits/rejected": -1.6292721033096313,
      "logps/chosen": -230.958740234375,
      "logps/rejected": -332.935791015625,
      "loss": 0.146,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.135877251625061,
      "rewards/margins": 5.311655521392822,
      "rewards/rejected": -6.447532653808594,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 3.1088597774505615,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.6871414184570312,
      "logits/rejected": -1.6046022176742554,
      "logps/chosen": -258.32257080078125,
      "logps/rejected": -330.11175537109375,
      "loss": 0.1706,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.5846526622772217,
      "rewards/margins": 5.45190954208374,
      "rewards/rejected": -7.036561489105225,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 5.220170974731445,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.7173973321914673,
      "logits/rejected": -1.6032285690307617,
      "logps/chosen": -216.53848266601562,
      "logps/rejected": -333.8974609375,
      "loss": 0.1272,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.540093183517456,
      "rewards/margins": 5.554872035980225,
      "rewards/rejected": -7.09496545791626,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 3.8425443172454834,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.705578088760376,
      "logits/rejected": -1.5608021020889282,
      "logps/chosen": -242.1476593017578,
      "logps/rejected": -323.027587890625,
      "loss": 0.1314,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.3977354764938354,
      "rewards/margins": 5.5933837890625,
      "rewards/rejected": -6.991118431091309,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 5.495195388793945,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.7403780221939087,
      "logits/rejected": -1.596381425857544,
      "logps/chosen": -233.91488647460938,
      "logps/rejected": -321.4733581542969,
      "loss": 0.1461,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7344648838043213,
      "rewards/margins": 5.078824996948242,
      "rewards/rejected": -6.813290596008301,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 2.574321746826172,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.8257449865341187,
      "logits/rejected": -1.6127040386199951,
      "logps/chosen": -254.885009765625,
      "logps/rejected": -340.189453125,
      "loss": 0.1662,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -1.7457427978515625,
      "rewards/margins": 5.908649444580078,
      "rewards/rejected": -7.654392242431641,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 3.7627923488616943,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.7805149555206299,
      "logits/rejected": -1.5877693891525269,
      "logps/chosen": -255.69650268554688,
      "logps/rejected": -325.4459228515625,
      "loss": 0.1219,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.5994096994400024,
      "rewards/margins": 5.30047607421875,
      "rewards/rejected": -6.899886131286621,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 4.437411785125732,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.8212392330169678,
      "logits/rejected": -1.7462043762207031,
      "logps/chosen": -227.9088897705078,
      "logps/rejected": -346.3284912109375,
      "loss": 0.1388,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6472021341323853,
      "rewards/margins": 6.0712714195251465,
      "rewards/rejected": -7.718472957611084,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 3.5579206943511963,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.7327353954315186,
      "logits/rejected": -1.5691357851028442,
      "logps/chosen": -235.33078002929688,
      "logps/rejected": -324.94476318359375,
      "loss": 0.1518,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.870516061782837,
      "rewards/margins": 5.5859222412109375,
      "rewards/rejected": -7.4564385414123535,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 3.057058095932007,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.7148669958114624,
      "logits/rejected": -1.6496082544326782,
      "logps/chosen": -233.3097381591797,
      "logps/rejected": -328.40191650390625,
      "loss": 0.1543,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.7619035243988037,
      "rewards/margins": 5.8997483253479,
      "rewards/rejected": -7.661650657653809,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 3.640073299407959,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.813275933265686,
      "logits/rejected": -1.7277463674545288,
      "logps/chosen": -225.76553344726562,
      "logps/rejected": -361.9228820800781,
      "loss": 0.1072,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.7821929454803467,
      "rewards/margins": 6.15872859954834,
      "rewards/rejected": -7.940920829772949,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 1.9339507818222046,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.7263004779815674,
      "logits/rejected": -1.6594091653823853,
      "logps/chosen": -220.85653686523438,
      "logps/rejected": -345.27886962890625,
      "loss": 0.121,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.9338741302490234,
      "rewards/margins": 5.8638129234313965,
      "rewards/rejected": -7.797686576843262,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 4.9970011711120605,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.6363716125488281,
      "logits/rejected": -1.5276905298233032,
      "logps/chosen": -232.1802215576172,
      "logps/rejected": -340.8988952636719,
      "loss": 0.1607,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.3411585092544556,
      "rewards/margins": 5.480500221252441,
      "rewards/rejected": -6.821658134460449,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 2.177555799484253,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.7766189575195312,
      "logits/rejected": -1.6084314584732056,
      "logps/chosen": -240.9921417236328,
      "logps/rejected": -330.74853515625,
      "loss": 0.1677,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.3653053045272827,
      "rewards/margins": 5.720976829528809,
      "rewards/rejected": -7.086282253265381,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": 1.8586421012878418,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.6043384075164795,
      "logits/rejected": -1.5343188047409058,
      "logps/chosen": -251.8693084716797,
      "logps/rejected": -341.179931640625,
      "loss": 0.1193,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6754684448242188,
      "rewards/margins": 5.657227516174316,
      "rewards/rejected": -7.332695960998535,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 2.7305967807769775,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.7863848209381104,
      "logits/rejected": -1.7329514026641846,
      "logps/chosen": -242.52517700195312,
      "logps/rejected": -347.587646484375,
      "loss": 0.1191,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.808240294456482,
      "rewards/margins": 5.8397536277771,
      "rewards/rejected": -7.647993564605713,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 3.4223201274871826,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.7167861461639404,
      "logits/rejected": -1.6005874872207642,
      "logps/chosen": -232.20309448242188,
      "logps/rejected": -352.6765441894531,
      "loss": 0.1321,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9672311544418335,
      "rewards/margins": 5.907352447509766,
      "rewards/rejected": -7.874582767486572,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 4.760759353637695,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.6677753925323486,
      "logits/rejected": -1.6088497638702393,
      "logps/chosen": -229.5364990234375,
      "logps/rejected": -340.87298583984375,
      "loss": 0.1338,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.956604242324829,
      "rewards/margins": 6.195125579833984,
      "rewards/rejected": -8.15173053741455,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 5.057645797729492,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.7027199268341064,
      "logits/rejected": -1.6212679147720337,
      "logps/chosen": -242.75393676757812,
      "logps/rejected": -342.66180419921875,
      "loss": 0.1214,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.713690996170044,
      "rewards/margins": 6.617132663726807,
      "rewards/rejected": -8.33082389831543,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 4.691673755645752,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.7766993045806885,
      "logits/rejected": -1.6785423755645752,
      "logps/chosen": -224.8117218017578,
      "logps/rejected": -356.70135498046875,
      "loss": 0.1368,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9061094522476196,
      "rewards/margins": 5.97117805480957,
      "rewards/rejected": -7.877286434173584,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 2.651674270629883,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.7267783880233765,
      "logits/rejected": -1.591758131980896,
      "logps/chosen": -245.8301544189453,
      "logps/rejected": -348.2117919921875,
      "loss": 0.1385,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.353121042251587,
      "rewards/margins": 5.891472816467285,
      "rewards/rejected": -8.244592666625977,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 3.751432418823242,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.7399965524673462,
      "logits/rejected": -1.636944055557251,
      "logps/chosen": -217.2939453125,
      "logps/rejected": -329.7176513671875,
      "loss": 0.1016,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9745547771453857,
      "rewards/margins": 6.236304759979248,
      "rewards/rejected": -8.210859298706055,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 3.8177692890167236,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.6969635486602783,
      "logits/rejected": -1.6023454666137695,
      "logps/chosen": -243.0435028076172,
      "logps/rejected": -339.3236389160156,
      "loss": 0.1492,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8978859186172485,
      "rewards/margins": 5.849517345428467,
      "rewards/rejected": -7.747402191162109,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 2.7171826362609863,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.7444900274276733,
      "logits/rejected": -1.5893446207046509,
      "logps/chosen": -228.6908416748047,
      "logps/rejected": -332.8255615234375,
      "loss": 0.1146,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6262632608413696,
      "rewards/margins": 6.102635383605957,
      "rewards/rejected": -7.7288994789123535,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 1.1198697090148926,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.7157386541366577,
      "logits/rejected": -1.6218187808990479,
      "logps/chosen": -251.6882781982422,
      "logps/rejected": -319.733154296875,
      "loss": 0.1369,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6356916427612305,
      "rewards/margins": 5.369327545166016,
      "rewards/rejected": -7.005019187927246,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 2.529115915298462,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.7237428426742554,
      "logits/rejected": -1.600476861000061,
      "logps/chosen": -245.54052734375,
      "logps/rejected": -338.63897705078125,
      "loss": 0.128,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.8269141912460327,
      "rewards/margins": 5.957518577575684,
      "rewards/rejected": -7.784433841705322,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 3.8999643325805664,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.6685173511505127,
      "logits/rejected": -1.5947197675704956,
      "logps/chosen": -249.70657348632812,
      "logps/rejected": -360.75921630859375,
      "loss": 0.133,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.994818091392517,
      "rewards/margins": 6.414160251617432,
      "rewards/rejected": -8.408979415893555,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 1.3885916471481323,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.7160930633544922,
      "logits/rejected": -1.6322555541992188,
      "logps/chosen": -229.70022583007812,
      "logps/rejected": -342.3292541503906,
      "loss": 0.1199,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6185846328735352,
      "rewards/margins": 6.081858158111572,
      "rewards/rejected": -7.700442314147949,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 1.6971677541732788,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.6721397638320923,
      "logits/rejected": -1.5602915287017822,
      "logps/chosen": -228.8137969970703,
      "logps/rejected": -322.86041259765625,
      "loss": 0.1142,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6498756408691406,
      "rewards/margins": 5.735910892486572,
      "rewards/rejected": -7.385787010192871,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 3.690915107727051,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.7795826196670532,
      "logits/rejected": -1.5900989770889282,
      "logps/chosen": -273.39202880859375,
      "logps/rejected": -340.42633056640625,
      "loss": 0.1149,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.4165130853652954,
      "rewards/margins": 6.7276177406311035,
      "rewards/rejected": -8.14413070678711,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 3.8674254417419434,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.6882655620574951,
      "logits/rejected": -1.6840887069702148,
      "logps/chosen": -229.9908905029297,
      "logps/rejected": -366.3896484375,
      "loss": 0.1271,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.206510066986084,
      "rewards/margins": 6.366598606109619,
      "rewards/rejected": -8.573108673095703,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 2.245314598083496,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.626455545425415,
      "logits/rejected": -1.5366604328155518,
      "logps/chosen": -252.74417114257812,
      "logps/rejected": -329.74664306640625,
      "loss": 0.1131,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.4865179061889648,
      "rewards/margins": 6.190736293792725,
      "rewards/rejected": -7.677253723144531,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 4.231085777282715,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.7769323587417603,
      "logits/rejected": -1.6287139654159546,
      "logps/chosen": -236.2292022705078,
      "logps/rejected": -332.585205078125,
      "loss": 0.1527,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8004322052001953,
      "rewards/margins": 5.81653356552124,
      "rewards/rejected": -7.616966247558594,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 1.3001052141189575,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.7131919860839844,
      "logits/rejected": -1.6334612369537354,
      "logps/chosen": -216.60595703125,
      "logps/rejected": -348.07049560546875,
      "loss": 0.0907,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.4661469459533691,
      "rewards/margins": 6.375014781951904,
      "rewards/rejected": -7.841162204742432,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 3.4341273307800293,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.7027778625488281,
      "logits/rejected": -1.5170468091964722,
      "logps/chosen": -239.1169891357422,
      "logps/rejected": -320.68255615234375,
      "loss": 0.1244,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.5550756454467773,
      "rewards/margins": 5.473415851593018,
      "rewards/rejected": -7.028491020202637,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 2.6593456268310547,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.7209646701812744,
      "logits/rejected": -1.5738383531570435,
      "logps/chosen": -260.90423583984375,
      "logps/rejected": -323.51531982421875,
      "loss": 0.1114,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1319111585617065,
      "rewards/margins": 5.710280418395996,
      "rewards/rejected": -6.84219217300415,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 4.085716247558594,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.768932580947876,
      "logits/rejected": -1.6225475072860718,
      "logps/chosen": -286.230712890625,
      "logps/rejected": -332.88458251953125,
      "loss": 0.1649,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.7526746988296509,
      "rewards/margins": 5.637965202331543,
      "rewards/rejected": -7.390639305114746,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 3.297569513320923,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.633087158203125,
      "logits/rejected": -1.5306732654571533,
      "logps/chosen": -253.0421600341797,
      "logps/rejected": -369.7610778808594,
      "loss": 0.1162,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5865943431854248,
      "rewards/margins": 6.107217311859131,
      "rewards/rejected": -7.693811893463135,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 5.6189866065979,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.793038010597229,
      "logits/rejected": -1.600799560546875,
      "logps/chosen": -233.15902709960938,
      "logps/rejected": -322.8293762207031,
      "loss": 0.1484,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6604406833648682,
      "rewards/margins": 5.723924160003662,
      "rewards/rejected": -7.384364128112793,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 5.962138652801514,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.7385631799697876,
      "logits/rejected": -1.6375945806503296,
      "logps/chosen": -235.4108428955078,
      "logps/rejected": -348.2346496582031,
      "loss": 0.1628,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -2.211745500564575,
      "rewards/margins": 5.744502067565918,
      "rewards/rejected": -7.956246852874756,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 3.5429904460906982,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.720656394958496,
      "logits/rejected": -1.5785796642303467,
      "logps/chosen": -229.98727416992188,
      "logps/rejected": -335.8892822265625,
      "loss": 0.1678,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8988584280014038,
      "rewards/margins": 6.257178783416748,
      "rewards/rejected": -8.156037330627441,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 4.259081840515137,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.7053463459014893,
      "logits/rejected": -1.6394329071044922,
      "logps/chosen": -247.9888458251953,
      "logps/rejected": -358.6705017089844,
      "loss": 0.0996,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9432106018066406,
      "rewards/margins": 6.078979969024658,
      "rewards/rejected": -8.022192001342773,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 1.0865824222564697,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.727698564529419,
      "logits/rejected": -1.6279067993164062,
      "logps/chosen": -244.95751953125,
      "logps/rejected": -347.68206787109375,
      "loss": 0.1208,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9562523365020752,
      "rewards/margins": 5.863669395446777,
      "rewards/rejected": -7.81992244720459,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 3.200122356414795,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.685605764389038,
      "logits/rejected": -1.5968866348266602,
      "logps/chosen": -233.25741577148438,
      "logps/rejected": -340.8912658691406,
      "loss": 0.1096,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.931910753250122,
      "rewards/margins": 6.27100944519043,
      "rewards/rejected": -8.202919006347656,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 2.352064609527588,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.71112859249115,
      "logits/rejected": -1.65338933467865,
      "logps/chosen": -251.2657012939453,
      "logps/rejected": -346.6195373535156,
      "loss": 0.1354,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.7670342922210693,
      "rewards/margins": 6.0953803062438965,
      "rewards/rejected": -7.862415313720703,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 3.149423599243164,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.730526328086853,
      "logits/rejected": -1.6701030731201172,
      "logps/chosen": -244.6688995361328,
      "logps/rejected": -354.5580139160156,
      "loss": 0.1045,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.046992063522339,
      "rewards/margins": 6.459168910980225,
      "rewards/rejected": -8.506160736083984,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 3.0293586254119873,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.8239294290542603,
      "logits/rejected": -1.7257106304168701,
      "logps/chosen": -219.5343475341797,
      "logps/rejected": -334.0470275878906,
      "loss": 0.0864,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.1547927856445312,
      "rewards/margins": 6.76791524887085,
      "rewards/rejected": -7.922706604003906,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 1.0889246463775635,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.7029361724853516,
      "logits/rejected": -1.629794716835022,
      "logps/chosen": -241.243408203125,
      "logps/rejected": -357.905029296875,
      "loss": 0.0971,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.594839334487915,
      "rewards/margins": 6.8463897705078125,
      "rewards/rejected": -8.441228866577148,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 3.799083948135376,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.6510183811187744,
      "logits/rejected": -1.6240144968032837,
      "logps/chosen": -243.8262481689453,
      "logps/rejected": -349.89569091796875,
      "loss": 0.1311,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8060070276260376,
      "rewards/margins": 5.853053092956543,
      "rewards/rejected": -7.659060001373291,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 2.7306034564971924,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.7491909265518188,
      "logits/rejected": -1.6059234142303467,
      "logps/chosen": -239.9043426513672,
      "logps/rejected": -348.3580322265625,
      "loss": 0.1165,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.591201901435852,
      "rewards/margins": 6.327251434326172,
      "rewards/rejected": -7.918453216552734,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 2.9349820613861084,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.7439231872558594,
      "logits/rejected": -1.6497364044189453,
      "logps/chosen": -235.9754638671875,
      "logps/rejected": -345.97601318359375,
      "loss": 0.1038,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5834319591522217,
      "rewards/margins": 6.660479545593262,
      "rewards/rejected": -8.243912696838379,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 0.882309079170227,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.7273489236831665,
      "logits/rejected": -1.6305004358291626,
      "logps/chosen": -225.79464721679688,
      "logps/rejected": -357.1151428222656,
      "loss": 0.0636,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.861360788345337,
      "rewards/margins": 6.6892828941345215,
      "rewards/rejected": -8.550643920898438,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 2.656388759613037,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.6046922206878662,
      "logits/rejected": -1.5097949504852295,
      "logps/chosen": -230.23007202148438,
      "logps/rejected": -341.9237365722656,
      "loss": 0.1282,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.8171331882476807,
      "rewards/margins": 6.508596897125244,
      "rewards/rejected": -8.32573127746582,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 1.9275213479995728,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.6167736053466797,
      "logits/rejected": -1.5170481204986572,
      "logps/chosen": -273.21685791015625,
      "logps/rejected": -357.8608093261719,
      "loss": 0.0853,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7052383422851562,
      "rewards/margins": 6.317803859710693,
      "rewards/rejected": -8.023042678833008,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 4.2159342765808105,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.6312297582626343,
      "logits/rejected": -1.6414638757705688,
      "logps/chosen": -265.60101318359375,
      "logps/rejected": -363.58709716796875,
      "loss": 0.113,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.922377586364746,
      "rewards/margins": 6.6036553382873535,
      "rewards/rejected": -8.526033401489258,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 5.185214519500732,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.6335700750350952,
      "logits/rejected": -1.6021028757095337,
      "logps/chosen": -260.4001770019531,
      "logps/rejected": -361.32861328125,
      "loss": 0.1311,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.589746356010437,
      "rewards/margins": 6.472695827484131,
      "rewards/rejected": -8.062442779541016,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 4.318290710449219,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.7304105758666992,
      "logits/rejected": -1.580280065536499,
      "logps/chosen": -233.4041748046875,
      "logps/rejected": -334.1758728027344,
      "loss": 0.1128,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.5012531280517578,
      "rewards/margins": 6.586421966552734,
      "rewards/rejected": -8.087675094604492,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 3.5980780124664307,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.761781930923462,
      "logits/rejected": -1.5910484790802002,
      "logps/chosen": -235.13815307617188,
      "logps/rejected": -339.7412109375,
      "loss": 0.1096,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9504568576812744,
      "rewards/margins": 6.570442199707031,
      "rewards/rejected": -8.520899772644043,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 5.879115104675293,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.723870038986206,
      "logits/rejected": -1.5144500732421875,
      "logps/chosen": -255.23355102539062,
      "logps/rejected": -321.55352783203125,
      "loss": 0.1381,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5773289203643799,
      "rewards/margins": 5.967000484466553,
      "rewards/rejected": -7.544329643249512,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 3.6410043239593506,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6668065786361694,
      "logits/rejected": -1.5639894008636475,
      "logps/chosen": -259.15362548828125,
      "logps/rejected": -351.24542236328125,
      "loss": 0.1139,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8555443286895752,
      "rewards/margins": 6.0399627685546875,
      "rewards/rejected": -7.895506858825684,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 3.0186233520507812,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.755916953086853,
      "logits/rejected": -1.6539405584335327,
      "logps/chosen": -265.0111389160156,
      "logps/rejected": -341.22918701171875,
      "loss": 0.0748,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.075282573699951,
      "rewards/margins": 6.287680149078369,
      "rewards/rejected": -8.36296272277832,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 2.5974314212799072,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.7911674976348877,
      "logits/rejected": -1.738271951675415,
      "logps/chosen": -243.0534210205078,
      "logps/rejected": -364.8657531738281,
      "loss": 0.0822,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5279881954193115,
      "rewards/margins": 6.4855852127075195,
      "rewards/rejected": -8.013572692871094,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 4.313776016235352,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.6735479831695557,
      "logits/rejected": -1.6453258991241455,
      "logps/chosen": -229.39956665039062,
      "logps/rejected": -348.58221435546875,
      "loss": 0.0763,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9789535999298096,
      "rewards/margins": 7.005384922027588,
      "rewards/rejected": -8.984338760375977,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 2.7341182231903076,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.6279489994049072,
      "logits/rejected": -1.5178601741790771,
      "logps/chosen": -255.06033325195312,
      "logps/rejected": -350.7305603027344,
      "loss": 0.1321,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.8026683330535889,
      "rewards/margins": 6.296760559082031,
      "rewards/rejected": -8.099428176879883,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 1.8301348686218262,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.5837254524230957,
      "logits/rejected": -1.5579044818878174,
      "logps/chosen": -235.1605987548828,
      "logps/rejected": -323.0276794433594,
      "loss": 0.1251,
      "rewards/accuracies": 0.9539473652839661,
      "rewards/chosen": -2.1340339183807373,
      "rewards/margins": 5.8628339767456055,
      "rewards/rejected": -7.996868133544922,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 1.6975414752960205,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.6794497966766357,
      "logits/rejected": -1.6130616664886475,
      "logps/chosen": -244.8574676513672,
      "logps/rejected": -361.6617431640625,
      "loss": 0.0732,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9459739923477173,
      "rewards/margins": 7.019693851470947,
      "rewards/rejected": -8.965668678283691,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 0.8381558656692505,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.6198406219482422,
      "logits/rejected": -1.5704479217529297,
      "logps/chosen": -228.9700927734375,
      "logps/rejected": -338.58782958984375,
      "loss": 0.1104,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4630517959594727,
      "rewards/margins": 6.167015552520752,
      "rewards/rejected": -7.630067348480225,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 2.7961416244506836,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.7360506057739258,
      "logits/rejected": -1.616646409034729,
      "logps/chosen": -256.83477783203125,
      "logps/rejected": -338.1868896484375,
      "loss": 0.1565,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1831955909729004,
      "rewards/margins": 6.078934192657471,
      "rewards/rejected": -8.262128829956055,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 4.844041347503662,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.7123730182647705,
      "logits/rejected": -1.5840885639190674,
      "logps/chosen": -279.18988037109375,
      "logps/rejected": -340.1874694824219,
      "loss": 0.116,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7493995428085327,
      "rewards/margins": 6.935929298400879,
      "rewards/rejected": -8.68532943725586,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 1.0539467334747314,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.7058155536651611,
      "logits/rejected": -1.5903913974761963,
      "logps/chosen": -246.2003631591797,
      "logps/rejected": -343.7231750488281,
      "loss": 0.1024,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8906090259552002,
      "rewards/margins": 6.115625381469727,
      "rewards/rejected": -8.006235122680664,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 1.198693871498108,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.7139508724212646,
      "logits/rejected": -1.6323169469833374,
      "logps/chosen": -233.64096069335938,
      "logps/rejected": -358.6378479003906,
      "loss": 0.0696,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2217655181884766,
      "rewards/margins": 6.9201250076293945,
      "rewards/rejected": -9.141891479492188,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 1.8502142429351807,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.6855157613754272,
      "logits/rejected": -1.6282001733779907,
      "logps/chosen": -232.8811492919922,
      "logps/rejected": -367.14984130859375,
      "loss": 0.066,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.8243892192840576,
      "rewards/margins": 7.305993556976318,
      "rewards/rejected": -9.130383491516113,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 6.383020877838135,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.5834999084472656,
      "logits/rejected": -1.4955960512161255,
      "logps/chosen": -263.1512451171875,
      "logps/rejected": -324.0372314453125,
      "loss": 0.1725,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.092085123062134,
      "rewards/margins": 5.891361236572266,
      "rewards/rejected": -7.9834465980529785,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 4.000116348266602,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.6087535619735718,
      "logits/rejected": -1.558837890625,
      "logps/chosen": -256.40777587890625,
      "logps/rejected": -364.42132568359375,
      "loss": 0.0999,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2485008239746094,
      "rewards/margins": 6.468873500823975,
      "rewards/rejected": -8.717373847961426,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 4.535819053649902,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.7264890670776367,
      "logits/rejected": -1.6320692300796509,
      "logps/chosen": -239.48046875,
      "logps/rejected": -359.3309020996094,
      "loss": 0.1133,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8677961826324463,
      "rewards/margins": 6.378199577331543,
      "rewards/rejected": -8.245996475219727,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 2.3757009506225586,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.726922631263733,
      "logits/rejected": -1.6309759616851807,
      "logps/chosen": -239.71511840820312,
      "logps/rejected": -373.7091979980469,
      "loss": 0.0649,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3566739559173584,
      "rewards/margins": 7.294585227966309,
      "rewards/rejected": -9.651260375976562,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 1.9476866722106934,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.7482807636260986,
      "logits/rejected": -1.6069862842559814,
      "logps/chosen": -240.14382934570312,
      "logps/rejected": -361.3963623046875,
      "loss": 0.1027,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.955012559890747,
      "rewards/margins": 7.11974573135376,
      "rewards/rejected": -9.074758529663086,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 2.6776695251464844,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.621201753616333,
      "logits/rejected": -1.5100598335266113,
      "logps/chosen": -238.4880828857422,
      "logps/rejected": -348.99945068359375,
      "loss": 0.0706,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1084201335906982,
      "rewards/margins": 7.692069053649902,
      "rewards/rejected": -9.80048942565918,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 2.4083611965179443,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.6992480754852295,
      "logits/rejected": -1.6273291110992432,
      "logps/chosen": -256.5484924316406,
      "logps/rejected": -333.29315185546875,
      "loss": 0.0934,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9675493240356445,
      "rewards/margins": 6.3934197425842285,
      "rewards/rejected": -8.360968589782715,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 4.460314750671387,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.7661964893341064,
      "logits/rejected": -1.620145559310913,
      "logps/chosen": -249.1160125732422,
      "logps/rejected": -336.40936279296875,
      "loss": 0.1163,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3201756477355957,
      "rewards/margins": 6.340085506439209,
      "rewards/rejected": -8.660261154174805,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 6.481249809265137,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.6359132528305054,
      "logits/rejected": -1.5414669513702393,
      "logps/chosen": -249.36630249023438,
      "logps/rejected": -333.96807861328125,
      "loss": 0.137,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.8785040378570557,
      "rewards/margins": 5.66298770904541,
      "rewards/rejected": -7.541491508483887,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 1.7318475246429443,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.7494544982910156,
      "logits/rejected": -1.6931339502334595,
      "logps/chosen": -219.39083862304688,
      "logps/rejected": -379.05084228515625,
      "loss": 0.06,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0614876747131348,
      "rewards/margins": 7.624110221862793,
      "rewards/rejected": -9.685598373413086,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 2.786151885986328,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.6171119213104248,
      "logits/rejected": -1.507101058959961,
      "logps/chosen": -243.9236297607422,
      "logps/rejected": -336.0796813964844,
      "loss": 0.0979,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2703404426574707,
      "rewards/margins": 6.305020809173584,
      "rewards/rejected": -8.575361251831055,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 2.109221935272217,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.6782341003417969,
      "logits/rejected": -1.5760619640350342,
      "logps/chosen": -240.6625518798828,
      "logps/rejected": -367.6160888671875,
      "loss": 0.0783,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.0386109352111816,
      "rewards/margins": 7.059946537017822,
      "rewards/rejected": -9.09855842590332,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 2.13687801361084,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.6757415533065796,
      "logits/rejected": -1.6712865829467773,
      "logps/chosen": -230.51266479492188,
      "logps/rejected": -366.68133544921875,
      "loss": 0.0587,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3439688682556152,
      "rewards/margins": 7.779986381530762,
      "rewards/rejected": -10.123955726623535,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 2.922100067138672,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.704439401626587,
      "logits/rejected": -1.50922429561615,
      "logps/chosen": -253.18704223632812,
      "logps/rejected": -359.7574768066406,
      "loss": 0.1019,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.250194549560547,
      "rewards/margins": 7.344038486480713,
      "rewards/rejected": -9.594232559204102,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 2.7870895862579346,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.6217025518417358,
      "logits/rejected": -1.53660249710083,
      "logps/chosen": -238.7826690673828,
      "logps/rejected": -347.23980712890625,
      "loss": 0.0898,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.125499725341797,
      "rewards/margins": 6.903764247894287,
      "rewards/rejected": -9.029264450073242,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 2.5745162963867188,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.671149492263794,
      "logits/rejected": -1.5366013050079346,
      "logps/chosen": -253.76779174804688,
      "logps/rejected": -356.57720947265625,
      "loss": 0.0588,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3465628623962402,
      "rewards/margins": 7.49077844619751,
      "rewards/rejected": -9.837340354919434,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 5.530954837799072,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.709287405014038,
      "logits/rejected": -1.6433643102645874,
      "logps/chosen": -254.7502899169922,
      "logps/rejected": -364.1354675292969,
      "loss": 0.0799,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.653001070022583,
      "rewards/margins": 6.731104850769043,
      "rewards/rejected": -9.384105682373047,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 2.934730291366577,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.6000057458877563,
      "logits/rejected": -1.5496574640274048,
      "logps/chosen": -253.9340057373047,
      "logps/rejected": -345.54180908203125,
      "loss": 0.0853,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.3913419246673584,
      "rewards/margins": 6.566730499267578,
      "rewards/rejected": -8.958072662353516,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 4.504528999328613,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.6582725048065186,
      "logits/rejected": -1.6277124881744385,
      "logps/chosen": -243.41665649414062,
      "logps/rejected": -363.15008544921875,
      "loss": 0.1113,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.7339158058166504,
      "rewards/margins": 7.162438869476318,
      "rewards/rejected": -9.896354675292969,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 3.5266401767730713,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.569681167602539,
      "logits/rejected": -1.5099362134933472,
      "logps/chosen": -243.1770782470703,
      "logps/rejected": -348.07501220703125,
      "loss": 0.1,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.601651906967163,
      "rewards/margins": 6.440603733062744,
      "rewards/rejected": -9.042256355285645,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 4.0208740234375,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.6699787378311157,
      "logits/rejected": -1.5480045080184937,
      "logps/chosen": -257.97100830078125,
      "logps/rejected": -384.45721435546875,
      "loss": 0.0927,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.272108793258667,
      "rewards/margins": 7.204779624938965,
      "rewards/rejected": -9.476888656616211,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 4.031130790710449,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.6949599981307983,
      "logits/rejected": -1.6505162715911865,
      "logps/chosen": -245.27963256835938,
      "logps/rejected": -368.87347412109375,
      "loss": 0.0983,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8141303062438965,
      "rewards/margins": 7.234206199645996,
      "rewards/rejected": -10.04833698272705,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 3.7834320068359375,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.7620437145233154,
      "logits/rejected": -1.5727145671844482,
      "logps/chosen": -248.0930633544922,
      "logps/rejected": -334.4790954589844,
      "loss": 0.0838,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.3493759632110596,
      "rewards/margins": 6.401058197021484,
      "rewards/rejected": -8.750434875488281,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 3.1130168437957764,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.5150299072265625,
      "logits/rejected": -1.459728479385376,
      "logps/chosen": -247.00454711914062,
      "logps/rejected": -355.4990539550781,
      "loss": 0.114,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.584754228591919,
      "rewards/margins": 6.42316198348999,
      "rewards/rejected": -9.007917404174805,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 1.063434362411499,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.684495210647583,
      "logits/rejected": -1.6073039770126343,
      "logps/chosen": -249.10623168945312,
      "logps/rejected": -368.306884765625,
      "loss": 0.0974,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6043505668640137,
      "rewards/margins": 7.219007968902588,
      "rewards/rejected": -9.823358535766602,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 1.9829832315444946,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.6180429458618164,
      "logits/rejected": -1.6057523488998413,
      "logps/chosen": -250.98989868164062,
      "logps/rejected": -369.36737060546875,
      "loss": 0.0887,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9293911457061768,
      "rewards/margins": 6.682594299316406,
      "rewards/rejected": -9.61198616027832,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 2.474097728729248,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.7592651844024658,
      "logits/rejected": -1.7193400859832764,
      "logps/chosen": -249.07107543945312,
      "logps/rejected": -377.8904113769531,
      "loss": 0.081,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.1483211517333984,
      "rewards/margins": 7.270550727844238,
      "rewards/rejected": -10.41887092590332,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 2.6713147163391113,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.6553032398223877,
      "logits/rejected": -1.5625696182250977,
      "logps/chosen": -241.57406616210938,
      "logps/rejected": -369.3149108886719,
      "loss": 0.0719,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5870490074157715,
      "rewards/margins": 7.614378452301025,
      "rewards/rejected": -10.201428413391113,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 1.6205579042434692,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.5774857997894287,
      "logits/rejected": -1.534125566482544,
      "logps/chosen": -224.51943969726562,
      "logps/rejected": -351.11956787109375,
      "loss": 0.0673,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.335993528366089,
      "rewards/margins": 7.358288764953613,
      "rewards/rejected": -9.694281578063965,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 3.4620277881622314,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.7100372314453125,
      "logits/rejected": -1.6061903238296509,
      "logps/chosen": -233.5575714111328,
      "logps/rejected": -362.2322082519531,
      "loss": 0.0903,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3436803817749023,
      "rewards/margins": 7.102326393127441,
      "rewards/rejected": -9.446006774902344,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 2.0486199855804443,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.6620476245880127,
      "logits/rejected": -1.5756416320800781,
      "logps/chosen": -240.8001708984375,
      "logps/rejected": -382.3775329589844,
      "loss": 0.0946,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7751338481903076,
      "rewards/margins": 7.866480827331543,
      "rewards/rejected": -10.641613960266113,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 5.349671840667725,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.8110148906707764,
      "logits/rejected": -1.6730849742889404,
      "logps/chosen": -255.79312133789062,
      "logps/rejected": -365.10797119140625,
      "loss": 0.0957,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.783545970916748,
      "rewards/margins": 7.280545711517334,
      "rewards/rejected": -10.064092636108398,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 2.677881956100464,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.603020429611206,
      "logits/rejected": -1.5301001071929932,
      "logps/chosen": -259.9833068847656,
      "logps/rejected": -365.3714599609375,
      "loss": 0.0842,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8131003379821777,
      "rewards/margins": 6.671302795410156,
      "rewards/rejected": -9.484403610229492,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 1.704085350036621,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.7008240222930908,
      "logits/rejected": -1.6075012683868408,
      "logps/chosen": -255.2787322998047,
      "logps/rejected": -349.6066589355469,
      "loss": 0.1014,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.5429394245147705,
      "rewards/margins": 6.982499599456787,
      "rewards/rejected": -9.525439262390137,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 3.8598265647888184,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.7050069570541382,
      "logits/rejected": -1.6236718893051147,
      "logps/chosen": -230.6517333984375,
      "logps/rejected": -369.06280517578125,
      "loss": 0.0873,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.6450648307800293,
      "rewards/margins": 7.3477582931518555,
      "rewards/rejected": -9.992822647094727,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 1.6293611526489258,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.6456454992294312,
      "logits/rejected": -1.4503004550933838,
      "logps/chosen": -253.2928009033203,
      "logps/rejected": -352.13055419921875,
      "loss": 0.088,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.3692188262939453,
      "rewards/margins": 6.8818864822387695,
      "rewards/rejected": -9.251104354858398,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 1.0771186351776123,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.6047093868255615,
      "logits/rejected": -1.5732990503311157,
      "logps/chosen": -258.16510009765625,
      "logps/rejected": -393.47235107421875,
      "loss": 0.0635,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.996021270751953,
      "rewards/margins": 7.686588287353516,
      "rewards/rejected": -10.682609558105469,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 1.822938323020935,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.6440626382827759,
      "logits/rejected": -1.5163719654083252,
      "logps/chosen": -268.2124328613281,
      "logps/rejected": -355.87786865234375,
      "loss": 0.0713,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7518844604492188,
      "rewards/margins": 7.2615203857421875,
      "rewards/rejected": -10.013406753540039,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 3.44150447845459,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.6613085269927979,
      "logits/rejected": -1.5570452213287354,
      "logps/chosen": -239.2145233154297,
      "logps/rejected": -363.83465576171875,
      "loss": 0.0576,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.467777729034424,
      "rewards/margins": 7.381143093109131,
      "rewards/rejected": -9.848920822143555,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 1.4450147151947021,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.5924997329711914,
      "logits/rejected": -1.510423183441162,
      "logps/chosen": -245.48831176757812,
      "logps/rejected": -368.66912841796875,
      "loss": 0.0768,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4841134548187256,
      "rewards/margins": 7.180917263031006,
      "rewards/rejected": -9.665030479431152,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 3.4693479537963867,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6568119525909424,
      "logits/rejected": -1.596027135848999,
      "logps/chosen": -246.05126953125,
      "logps/rejected": -328.98858642578125,
      "loss": 0.0848,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.629823684692383,
      "rewards/margins": 6.541849613189697,
      "rewards/rejected": -9.171671867370605,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 1.5991712808609009,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.6681095361709595,
      "logits/rejected": -1.5474411249160767,
      "logps/chosen": -246.9559783935547,
      "logps/rejected": -365.996337890625,
      "loss": 0.0796,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.6049845218658447,
      "rewards/margins": 7.331271171569824,
      "rewards/rejected": -9.93625545501709,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.4303497076034546,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.7470356225967407,
      "logits/rejected": -1.6877567768096924,
      "logps/chosen": -237.14285278320312,
      "logps/rejected": -378.49774169921875,
      "loss": 0.0815,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6633784770965576,
      "rewards/margins": 7.212291717529297,
      "rewards/rejected": -9.875669479370117,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 1.2953486442565918,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.7139877080917358,
      "logits/rejected": -1.6032788753509521,
      "logps/chosen": -254.2081756591797,
      "logps/rejected": -370.1174621582031,
      "loss": 0.0741,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.655818223953247,
      "rewards/margins": 6.725550651550293,
      "rewards/rejected": -9.381368637084961,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 2.0400030612945557,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.7738536596298218,
      "logits/rejected": -1.5883662700653076,
      "logps/chosen": -240.4284210205078,
      "logps/rejected": -364.11614990234375,
      "loss": 0.0749,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.759413480758667,
      "rewards/margins": 7.233453273773193,
      "rewards/rejected": -9.992866516113281,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 2.236142635345459,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.6081100702285767,
      "logits/rejected": -1.5720396041870117,
      "logps/chosen": -266.195556640625,
      "logps/rejected": -372.3501281738281,
      "loss": 0.0692,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.3031420707702637,
      "rewards/margins": 7.687568664550781,
      "rewards/rejected": -9.990710258483887,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 1.1345685720443726,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.586785912513733,
      "logits/rejected": -1.547318696975708,
      "logps/chosen": -271.5514831542969,
      "logps/rejected": -361.0550842285156,
      "loss": 0.1134,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.704498767852783,
      "rewards/margins": 7.07074499130249,
      "rewards/rejected": -9.775243759155273,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 2.8865694999694824,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.6625696420669556,
      "logits/rejected": -1.5965254306793213,
      "logps/chosen": -244.66842651367188,
      "logps/rejected": -362.2137145996094,
      "loss": 0.0739,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.790815830230713,
      "rewards/margins": 7.540101528167725,
      "rewards/rejected": -10.330916404724121,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 1.5729948282241821,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.6541128158569336,
      "logits/rejected": -1.5849196910858154,
      "logps/chosen": -237.2561492919922,
      "logps/rejected": -365.4972839355469,
      "loss": 0.0791,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8733794689178467,
      "rewards/margins": 7.663126468658447,
      "rewards/rejected": -10.536505699157715,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 1.4792896509170532,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.6012680530548096,
      "logits/rejected": -1.5582574605941772,
      "logps/chosen": -234.076416015625,
      "logps/rejected": -345.57452392578125,
      "loss": 0.065,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5481960773468018,
      "rewards/margins": 6.975489616394043,
      "rewards/rejected": -9.523685455322266,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 1.1267932653427124,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.6668701171875,
      "logits/rejected": -1.5733473300933838,
      "logps/chosen": -257.2702331542969,
      "logps/rejected": -378.37188720703125,
      "loss": 0.1129,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.888535261154175,
      "rewards/margins": 7.025588035583496,
      "rewards/rejected": -9.91412353515625,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 1.5751965045928955,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.667828917503357,
      "logits/rejected": -1.5087411403656006,
      "logps/chosen": -256.654296875,
      "logps/rejected": -373.72021484375,
      "loss": 0.0712,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.67441463470459,
      "rewards/margins": 7.628767490386963,
      "rewards/rejected": -10.303182601928711,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 1.7919318675994873,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.6052993535995483,
      "logits/rejected": -1.5203253030776978,
      "logps/chosen": -277.48358154296875,
      "logps/rejected": -362.0965881347656,
      "loss": 0.0974,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6504619121551514,
      "rewards/margins": 6.992439270019531,
      "rewards/rejected": -9.642900466918945,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 2.0555083751678467,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.6291580200195312,
      "logits/rejected": -1.556952714920044,
      "logps/chosen": -239.12606811523438,
      "logps/rejected": -355.2477111816406,
      "loss": 0.1169,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -3.143803358078003,
      "rewards/margins": 7.427267551422119,
      "rewards/rejected": -10.571070671081543,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 2.651470184326172,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.6337629556655884,
      "logits/rejected": -1.583034873008728,
      "logps/chosen": -264.42333984375,
      "logps/rejected": -360.38616943359375,
      "loss": 0.0886,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7354702949523926,
      "rewards/margins": 7.126223087310791,
      "rewards/rejected": -9.861692428588867,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 2.948840618133545,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.6709390878677368,
      "logits/rejected": -1.5679157972335815,
      "logps/chosen": -299.9473571777344,
      "logps/rejected": -367.95245361328125,
      "loss": 0.077,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.1444687843322754,
      "rewards/margins": 7.233567714691162,
      "rewards/rejected": -9.378034591674805,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 2.0014069080352783,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.67729914188385,
      "logits/rejected": -1.5980799198150635,
      "logps/chosen": -247.76663208007812,
      "logps/rejected": -371.108154296875,
      "loss": 0.0757,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.236902952194214,
      "rewards/margins": 7.365381717681885,
      "rewards/rejected": -10.602285385131836,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 1.5738085508346558,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.6811107397079468,
      "logits/rejected": -1.5563424825668335,
      "logps/chosen": -246.9635772705078,
      "logps/rejected": -372.36859130859375,
      "loss": 0.0737,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.4004998207092285,
      "rewards/margins": 7.776610374450684,
      "rewards/rejected": -10.177111625671387,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 3.890855550765991,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.6264982223510742,
      "logits/rejected": -1.5898141860961914,
      "logps/chosen": -266.61248779296875,
      "logps/rejected": -345.47845458984375,
      "loss": 0.0846,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7661638259887695,
      "rewards/margins": 6.510016441345215,
      "rewards/rejected": -9.2761812210083,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 1.8012325763702393,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.6986353397369385,
      "logits/rejected": -1.5819010734558105,
      "logps/chosen": -250.3184051513672,
      "logps/rejected": -348.66973876953125,
      "loss": 0.0675,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.715459108352661,
      "rewards/margins": 6.96172571182251,
      "rewards/rejected": -9.67718505859375,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 2.982224464416504,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.5906603336334229,
      "logits/rejected": -1.5850645303726196,
      "logps/chosen": -247.61328125,
      "logps/rejected": -395.35711669921875,
      "loss": 0.1005,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.7781596183776855,
      "rewards/margins": 7.818112850189209,
      "rewards/rejected": -10.596271514892578,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 3.6533961296081543,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.6935789585113525,
      "logits/rejected": -1.602374792098999,
      "logps/chosen": -239.06887817382812,
      "logps/rejected": -341.3331604003906,
      "loss": 0.0929,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9050512313842773,
      "rewards/margins": 6.8136305809021,
      "rewards/rejected": -9.718682289123535,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 2.6796557903289795,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.6797822713851929,
      "logits/rejected": -1.6272222995758057,
      "logps/chosen": -241.6938934326172,
      "logps/rejected": -374.75958251953125,
      "loss": 0.0998,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -2.4390995502471924,
      "rewards/margins": 7.6869096755981445,
      "rewards/rejected": -10.126008987426758,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 1.120842456817627,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.5870236158370972,
      "logits/rejected": -1.5749309062957764,
      "logps/chosen": -243.9224090576172,
      "logps/rejected": -361.07000732421875,
      "loss": 0.0925,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.4479503631591797,
      "rewards/margins": 7.581417083740234,
      "rewards/rejected": -10.029366493225098,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 1.1996480226516724,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.5754233598709106,
      "logits/rejected": -1.5047509670257568,
      "logps/chosen": -256.6371765136719,
      "logps/rejected": -357.0195007324219,
      "loss": 0.0703,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.477647542953491,
      "rewards/margins": 7.629544734954834,
      "rewards/rejected": -10.107192039489746,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 0.44732823967933655,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.6828523874282837,
      "logits/rejected": -1.6784083843231201,
      "logps/chosen": -259.8722229003906,
      "logps/rejected": -372.0727844238281,
      "loss": 0.0545,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.719404697418213,
      "rewards/margins": 7.491476535797119,
      "rewards/rejected": -10.210882186889648,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 5.6885199546813965,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.6398189067840576,
      "logits/rejected": -1.587944746017456,
      "logps/chosen": -265.2816162109375,
      "logps/rejected": -378.03228759765625,
      "loss": 0.0913,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6671059131622314,
      "rewards/margins": 7.217464447021484,
      "rewards/rejected": -9.884570121765137,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 5.603551864624023,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.769521713256836,
      "logits/rejected": -1.6387608051300049,
      "logps/chosen": -257.2008056640625,
      "logps/rejected": -384.9828186035156,
      "loss": 0.0941,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.8595147132873535,
      "rewards/margins": 7.990152835845947,
      "rewards/rejected": -10.8496675491333,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 2.971264362335205,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.5761768817901611,
      "logits/rejected": -1.5605074167251587,
      "logps/chosen": -253.6317138671875,
      "logps/rejected": -371.7652587890625,
      "loss": 0.1064,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.9549922943115234,
      "rewards/margins": 7.519838809967041,
      "rewards/rejected": -10.474831581115723,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 4.489383697509766,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.6371389627456665,
      "logits/rejected": -1.5999717712402344,
      "logps/chosen": -253.1427459716797,
      "logps/rejected": -372.0256042480469,
      "loss": 0.0978,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.740422487258911,
      "rewards/margins": 6.826735496520996,
      "rewards/rejected": -9.567158699035645,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 2.8886446952819824,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.7825405597686768,
      "logits/rejected": -1.6278918981552124,
      "logps/chosen": -247.08450317382812,
      "logps/rejected": -344.13739013671875,
      "loss": 0.0729,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.63508677482605,
      "rewards/margins": 7.258586883544922,
      "rewards/rejected": -9.893674850463867,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 2.435647964477539,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.6750576496124268,
      "logits/rejected": -1.5368554592132568,
      "logps/chosen": -243.14306640625,
      "logps/rejected": -371.97296142578125,
      "loss": 0.0935,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.809309482574463,
      "rewards/margins": 7.868110656738281,
      "rewards/rejected": -10.677419662475586,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 1.5942474603652954,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.5710580348968506,
      "logits/rejected": -1.435702919960022,
      "logps/chosen": -283.32379150390625,
      "logps/rejected": -348.6867980957031,
      "loss": 0.0785,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.5561439990997314,
      "rewards/margins": 6.856298923492432,
      "rewards/rejected": -9.412443161010742,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 2.419877529144287,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.555304765701294,
      "logits/rejected": -1.5223093032836914,
      "logps/chosen": -255.93295288085938,
      "logps/rejected": -367.8576354980469,
      "loss": 0.1082,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8896031379699707,
      "rewards/margins": 7.200318813323975,
      "rewards/rejected": -10.089921951293945,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 3.1487884521484375,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.6507923603057861,
      "logits/rejected": -1.558606505393982,
      "logps/chosen": -251.55078125,
      "logps/rejected": -367.0152893066406,
      "loss": 0.0617,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1786768436431885,
      "rewards/margins": 7.703708648681641,
      "rewards/rejected": -10.882387161254883,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 3.5375068187713623,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.5756328105926514,
      "logits/rejected": -1.562476396560669,
      "logps/chosen": -259.0333557128906,
      "logps/rejected": -377.33734130859375,
      "loss": 0.1057,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.788533926010132,
      "rewards/margins": 7.0015106201171875,
      "rewards/rejected": -9.790044784545898,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 1.4896981716156006,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.6661159992218018,
      "logits/rejected": -1.6331936120986938,
      "logps/chosen": -235.8148193359375,
      "logps/rejected": -373.57183837890625,
      "loss": 0.1032,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9084439277648926,
      "rewards/margins": 7.1836676597595215,
      "rewards/rejected": -10.092111587524414,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 3.489358901977539,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.667912483215332,
      "logits/rejected": -1.5959059000015259,
      "logps/chosen": -257.36187744140625,
      "logps/rejected": -358.2672424316406,
      "loss": 0.0731,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.9006903171539307,
      "rewards/margins": 7.321658134460449,
      "rewards/rejected": -10.222349166870117,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 2.710655450820923,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.6697155237197876,
      "logits/rejected": -1.580635666847229,
      "logps/chosen": -233.53555297851562,
      "logps/rejected": -360.79888916015625,
      "loss": 0.1151,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -2.5065906047821045,
      "rewards/margins": 7.7552170753479,
      "rewards/rejected": -10.261807441711426,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 3.9822990894317627,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6743650436401367,
      "logits/rejected": -1.585886001586914,
      "logps/chosen": -264.65167236328125,
      "logps/rejected": -376.71588134765625,
      "loss": 0.0825,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1264894008636475,
      "rewards/margins": 7.663410186767578,
      "rewards/rejected": -10.789899826049805,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 2.6295809745788574,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6660277843475342,
      "logits/rejected": -1.558496356010437,
      "logps/chosen": -252.54244995117188,
      "logps/rejected": -352.0909118652344,
      "loss": 0.1049,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.452617883682251,
      "rewards/margins": 7.0967607498168945,
      "rewards/rejected": -9.549378395080566,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 1.3974653482437134,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.5303410291671753,
      "logits/rejected": -1.5407088994979858,
      "logps/chosen": -245.9228515625,
      "logps/rejected": -395.0731201171875,
      "loss": 0.0716,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.417079448699951,
      "rewards/margins": 7.852843284606934,
      "rewards/rejected": -10.269923210144043,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 2.3230302333831787,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.5849785804748535,
      "logits/rejected": -1.5197336673736572,
      "logps/chosen": -256.17095947265625,
      "logps/rejected": -375.9013671875,
      "loss": 0.0673,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8717899322509766,
      "rewards/margins": 7.789064884185791,
      "rewards/rejected": -10.660855293273926,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 4.476577281951904,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.595900297164917,
      "logits/rejected": -1.4698235988616943,
      "logps/chosen": -250.3801727294922,
      "logps/rejected": -344.8222351074219,
      "loss": 0.116,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.773193836212158,
      "rewards/margins": 7.142364501953125,
      "rewards/rejected": -9.915558815002441,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 2.3721888065338135,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.6179622411727905,
      "logits/rejected": -1.557403326034546,
      "logps/chosen": -276.74114990234375,
      "logps/rejected": -369.22088623046875,
      "loss": 0.1017,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8413569927215576,
      "rewards/margins": 7.1871490478515625,
      "rewards/rejected": -10.028505325317383,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 2.667341470718384,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.5483349561691284,
      "logits/rejected": -1.463370680809021,
      "logps/chosen": -256.5998229980469,
      "logps/rejected": -369.9665832519531,
      "loss": 0.0884,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8231709003448486,
      "rewards/margins": 7.097599983215332,
      "rewards/rejected": -9.920769691467285,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}