{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1407,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.010666666666666666,
      "grad_norm": 5.690470218658447,
      "learning_rate": 4.651162790697675e-07,
      "logits/chosen": -1.3886802196502686,
      "logits/rejected": -1.288570761680603,
      "logps/chosen": -198.13186645507812,
      "logps/rejected": -261.1497802734375,
      "loss": 0.6931,
      "rewards/accuracies": 0.3125,
      "rewards/chosen": -0.00018911839288193733,
      "rewards/margins": 0.00016672378114890307,
      "rewards/rejected": -0.0003558421158231795,
      "step": 5
    },
    {
      "epoch": 0.021333333333333333,
      "grad_norm": 2.7565765380859375,
      "learning_rate": 1.0465116279069768e-06,
      "logits/chosen": -1.3378444910049438,
      "logits/rejected": -1.2759315967559814,
      "logps/chosen": -206.43960571289062,
      "logps/rejected": -279.51348876953125,
      "loss": 0.691,
      "rewards/accuracies": 0.5874999761581421,
      "rewards/chosen": 0.001392951002344489,
      "rewards/margins": 0.0042605213820934296,
      "rewards/rejected": -0.002867569914087653,
      "step": 10
    },
    {
      "epoch": 0.032,
      "grad_norm": 3.2212350368499756,
      "learning_rate": 1.6279069767441862e-06,
      "logits/chosen": -1.3505547046661377,
      "logits/rejected": -1.2644460201263428,
      "logps/chosen": -206.5700225830078,
      "logps/rejected": -259.0060729980469,
      "loss": 0.6853,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 0.0011921453988179564,
      "rewards/margins": 0.015914104878902435,
      "rewards/rejected": -0.01472195703536272,
      "step": 15
    },
    {
      "epoch": 0.042666666666666665,
      "grad_norm": NaN,
      "learning_rate": 2.2093023255813954e-06,
      "logits/chosen": -1.4066317081451416,
      "logits/rejected": -1.2923839092254639,
      "logps/chosen": -223.789306640625,
      "logps/rejected": -258.15020751953125,
      "loss": 0.6749,
      "rewards/accuracies": 0.75,
      "rewards/chosen": 0.0023522863630205393,
      "rewards/margins": 0.03771216794848442,
      "rewards/rejected": -0.035359881818294525,
      "step": 20
    },
    {
      "epoch": 0.05333333333333334,
      "grad_norm": 2.9260594844818115,
      "learning_rate": 2.790697674418605e-06,
      "logits/chosen": -1.350759506225586,
      "logits/rejected": -1.16400146484375,
      "logps/chosen": -203.46731567382812,
      "logps/rejected": -283.2779235839844,
      "loss": 0.658,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0039240168407559395,
      "rewards/margins": 0.07307498157024384,
      "rewards/rejected": -0.0769990086555481,
      "step": 25
    },
    {
      "epoch": 0.064,
      "grad_norm": 3.8544366359710693,
      "learning_rate": 3.372093023255814e-06,
      "logits/chosen": -1.4426774978637695,
      "logits/rejected": -1.3100595474243164,
      "logps/chosen": -264.96258544921875,
      "logps/rejected": -255.35922241210938,
      "loss": 0.6423,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.01604917272925377,
      "rewards/margins": 0.10922543704509735,
      "rewards/rejected": -0.12527461349964142,
      "step": 30
    },
    {
      "epoch": 0.07466666666666667,
      "grad_norm": 2.8372976779937744,
      "learning_rate": 3.953488372093024e-06,
      "logits/chosen": -1.488498568534851,
      "logits/rejected": -1.4097141027450562,
      "logps/chosen": -210.7440643310547,
      "logps/rejected": -289.41046142578125,
      "loss": 0.6216,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.030715733766555786,
      "rewards/margins": 0.15629956126213074,
      "rewards/rejected": -0.1870153248310089,
      "step": 35
    },
    {
      "epoch": 0.08533333333333333,
      "grad_norm": 2.8906612396240234,
      "learning_rate": 4.5348837209302326e-06,
      "logits/chosen": -1.4510371685028076,
      "logits/rejected": -1.288226842880249,
      "logps/chosen": -207.5216064453125,
      "logps/rejected": -260.95416259765625,
      "loss": 0.6082,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.0654064267873764,
      "rewards/margins": 0.19735947251319885,
      "rewards/rejected": -0.26276591420173645,
      "step": 40
    },
    {
      "epoch": 0.096,
      "grad_norm": 2.679903984069824,
      "learning_rate": 4.99999336897255e-06,
      "logits/chosen": -1.4529269933700562,
      "logits/rejected": -1.2380075454711914,
      "logps/chosen": -211.7548370361328,
      "logps/rejected": -255.37753295898438,
      "loss": 0.5597,
      "rewards/accuracies": 0.8187500238418579,
      "rewards/chosen": -0.07748017460107803,
      "rewards/margins": 0.32700955867767334,
      "rewards/rejected": -0.4044896960258484,
      "step": 45
    },
    {
      "epoch": 0.10666666666666667,
      "grad_norm": 5.634088039398193,
      "learning_rate": 4.999761286705285e-06,
      "logits/chosen": -1.4369691610336304,
      "logits/rejected": -1.2739875316619873,
      "logps/chosen": -221.0635528564453,
      "logps/rejected": -271.1972351074219,
      "loss": 0.5247,
      "rewards/accuracies": 0.84375,
      "rewards/chosen": -0.11849220842123032,
      "rewards/margins": 0.4242885112762451,
      "rewards/rejected": -0.5427806973457336,
      "step": 50
    },
    {
      "epoch": 0.11733333333333333,
      "grad_norm": 2.9434328079223633,
      "learning_rate": 4.999197688241076e-06,
      "logits/chosen": -1.5113855600357056,
      "logits/rejected": -1.3324053287506104,
      "logps/chosen": -202.31285095214844,
      "logps/rejected": -265.8841552734375,
      "loss": 0.491,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.10869394242763519,
      "rewards/margins": 0.5555111765861511,
      "rewards/rejected": -0.6642050743103027,
      "step": 55
    },
    {
      "epoch": 0.128,
      "grad_norm": 3.161414623260498,
      "learning_rate": 4.9983026483238685e-06,
      "logits/chosen": -1.438110113143921,
      "logits/rejected": -1.2655045986175537,
      "logps/chosen": -211.4470977783203,
      "logps/rejected": -260.45208740234375,
      "loss": 0.4723,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": -0.2564672529697418,
      "rewards/margins": 0.6585448980331421,
      "rewards/rejected": -0.9150121808052063,
      "step": 60
    },
    {
      "epoch": 0.13866666666666666,
      "grad_norm": 2.4061851501464844,
      "learning_rate": 4.997076285653087e-06,
      "logits/chosen": -1.5721609592437744,
      "logits/rejected": -1.3645689487457275,
      "logps/chosen": -247.45956420898438,
      "logps/rejected": -268.1156921386719,
      "loss": 0.5025,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": -0.37047362327575684,
      "rewards/margins": 0.6632369756698608,
      "rewards/rejected": -1.0337107181549072,
      "step": 65
    },
    {
      "epoch": 0.14933333333333335,
      "grad_norm": 3.2865662574768066,
      "learning_rate": 4.995518762867897e-06,
      "logits/chosen": -1.4936473369598389,
      "logits/rejected": -1.2969480752944946,
      "logps/chosen": -206.06124877929688,
      "logps/rejected": -283.84466552734375,
      "loss": 0.4005,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.36351141333580017,
      "rewards/margins": 0.9965918660163879,
      "rewards/rejected": -1.3601032495498657,
      "step": 70
    },
    {
      "epoch": 0.16,
      "grad_norm": 2.552584409713745,
      "learning_rate": 4.993630286525634e-06,
      "logits/chosen": -1.548957109451294,
      "logits/rejected": -1.4676214456558228,
      "logps/chosen": -203.4427947998047,
      "logps/rejected": -263.97698974609375,
      "loss": 0.414,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.3925500214099884,
      "rewards/margins": 0.9759253263473511,
      "rewards/rejected": -1.368475317955017,
      "step": 75
    },
    {
      "epoch": 0.17066666666666666,
      "grad_norm": 2.2154462337493896,
      "learning_rate": 4.99141110707441e-06,
      "logits/chosen": -1.4633119106292725,
      "logits/rejected": -1.3160654306411743,
      "logps/chosen": -214.0872039794922,
      "logps/rejected": -276.56048583984375,
      "loss": 0.3877,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.44031819701194763,
      "rewards/margins": 1.116113305091858,
      "rewards/rejected": -1.556431531906128,
      "step": 80
    },
    {
      "epoch": 0.18133333333333335,
      "grad_norm": 2.944103479385376,
      "learning_rate": 4.9888615188199e-06,
      "logits/chosen": -1.5001494884490967,
      "logits/rejected": -1.3682790994644165,
      "logps/chosen": -207.25057983398438,
      "logps/rejected": -285.5174560546875,
      "loss": 0.3776,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.530932605266571,
      "rewards/margins": 1.2580114603042603,
      "rewards/rejected": -1.7889440059661865,
      "step": 85
    },
    {
      "epoch": 0.192,
      "grad_norm": 2.180025339126587,
      "learning_rate": 4.9859818598863104e-06,
      "logits/chosen": -1.5337731838226318,
      "logits/rejected": -1.3553698062896729,
      "logps/chosen": -201.07247924804688,
      "logps/rejected": -264.13848876953125,
      "loss": 0.3508,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.4138107895851135,
      "rewards/margins": 1.3532263040542603,
      "rewards/rejected": -1.767037034034729,
      "step": 90
    },
    {
      "epoch": 0.20266666666666666,
      "grad_norm": 2.4048197269439697,
      "learning_rate": 4.982772512171538e-06,
      "logits/chosen": -1.6030489206314087,
      "logits/rejected": -1.501514196395874,
      "logps/chosen": -198.14962768554688,
      "logps/rejected": -262.2642517089844,
      "loss": 0.3704,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.35309070348739624,
      "rewards/margins": 1.3474071025848389,
      "rewards/rejected": -1.7004978656768799,
      "step": 95
    },
    {
      "epoch": 0.21333333333333335,
      "grad_norm": 3.966644287109375,
      "learning_rate": 4.979233901296523e-06,
      "logits/chosen": -1.5420665740966797,
      "logits/rejected": -1.4332940578460693,
      "logps/chosen": -196.133056640625,
      "logps/rejected": -273.1980285644531,
      "loss": 0.3125,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.418867826461792,
      "rewards/margins": 1.6799125671386719,
      "rewards/rejected": -2.0987801551818848,
      "step": 100
    },
    {
      "epoch": 0.224,
      "grad_norm": 3.5509183406829834,
      "learning_rate": 4.975366496548804e-06,
      "logits/chosen": -1.6272293329238892,
      "logits/rejected": -1.396296739578247,
      "logps/chosen": -218.2744140625,
      "logps/rejected": -290.47222900390625,
      "loss": 0.3072,
      "rewards/accuracies": 0.8687499761581421,
      "rewards/chosen": -0.4910121560096741,
      "rewards/margins": 1.7056410312652588,
      "rewards/rejected": -2.196653127670288,
      "step": 105
    },
    {
      "epoch": 0.23466666666666666,
      "grad_norm": 3.0346481800079346,
      "learning_rate": 4.971170810820279e-06,
      "logits/chosen": -1.5354888439178467,
      "logits/rejected": -1.4109222888946533,
      "logps/chosen": -227.25634765625,
      "logps/rejected": -285.1552734375,
      "loss": 0.3081,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.5940927267074585,
      "rewards/margins": 1.8030599355697632,
      "rewards/rejected": -2.397152900695801,
      "step": 110
    },
    {
      "epoch": 0.24533333333333332,
      "grad_norm": 2.6833956241607666,
      "learning_rate": 4.96664740053919e-06,
      "logits/chosen": -1.5940526723861694,
      "logits/rejected": -1.4896529912948608,
      "logps/chosen": -219.3295440673828,
      "logps/rejected": -300.86102294921875,
      "loss": 0.2949,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.6788660287857056,
      "rewards/margins": 1.8880093097686768,
      "rewards/rejected": -2.5668752193450928,
      "step": 115
    },
    {
      "epoch": 0.256,
      "grad_norm": 2.7038156986236572,
      "learning_rate": 4.961796865596328e-06,
      "logits/chosen": -1.5613062381744385,
      "logits/rejected": -1.3715859651565552,
      "logps/chosen": -203.54425048828125,
      "logps/rejected": -283.4847412109375,
      "loss": 0.2571,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.3901914358139038,
      "rewards/margins": 2.150984287261963,
      "rewards/rejected": -2.5411758422851562,
      "step": 120
    },
    {
      "epoch": 0.26666666666666666,
      "grad_norm": 3.027423858642578,
      "learning_rate": 4.956619849265473e-06,
      "logits/chosen": -1.6930328607559204,
      "logits/rejected": -1.507812261581421,
      "logps/chosen": -207.44992065429688,
      "logps/rejected": -286.91607666015625,
      "loss": 0.2844,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.6250053644180298,
      "rewards/margins": 1.9750416278839111,
      "rewards/rejected": -2.6000471115112305,
      "step": 125
    },
    {
      "epoch": 0.2773333333333333,
      "grad_norm": 1.8523262739181519,
      "learning_rate": 4.951117038118091e-06,
      "logits/chosen": -1.655664086341858,
      "logits/rejected": -1.4919339418411255,
      "logps/chosen": -208.61209106445312,
      "logps/rejected": -279.31695556640625,
      "loss": 0.2413,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.5459321737289429,
      "rewards/margins": 2.22091007232666,
      "rewards/rejected": -2.7668423652648926,
      "step": 130
    },
    {
      "epoch": 0.288,
      "grad_norm": 3.026885986328125,
      "learning_rate": 4.945289161932273e-06,
      "logits/chosen": -1.6513007879257202,
      "logits/rejected": -1.5145455598831177,
      "logps/chosen": -219.18704223632812,
      "logps/rejected": -293.86993408203125,
      "loss": 0.2555,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6282526254653931,
      "rewards/margins": 2.321913957595825,
      "rewards/rejected": -2.950166702270508,
      "step": 135
    },
    {
      "epoch": 0.2986666666666667,
      "grad_norm": 1.7478291988372803,
      "learning_rate": 4.93913699359596e-06,
      "logits/chosen": -1.662444829940796,
      "logits/rejected": -1.4403254985809326,
      "logps/chosen": -199.58511352539062,
      "logps/rejected": -296.8211364746094,
      "loss": 0.26,
      "rewards/accuracies": 0.893750011920929,
      "rewards/chosen": -0.5024666786193848,
      "rewards/margins": 2.4235363006591797,
      "rewards/rejected": -2.9260029792785645,
      "step": 140
    },
    {
      "epoch": 0.30933333333333335,
      "grad_norm": 2.777461528778076,
      "learning_rate": 4.9326613490044364e-06,
      "logits/chosen": -1.6423578262329102,
      "logits/rejected": -1.5219056606292725,
      "logps/chosen": -211.6616668701172,
      "logps/rejected": -273.14337158203125,
      "loss": 0.2553,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5515410304069519,
      "rewards/margins": 2.2121779918670654,
      "rewards/rejected": -2.763719081878662,
      "step": 145
    },
    {
      "epoch": 0.32,
      "grad_norm": 2.59855055809021,
      "learning_rate": 4.925863086952132e-06,
      "logits/chosen": -1.6805490255355835,
      "logits/rejected": -1.4335730075836182,
      "logps/chosen": -219.8978729248047,
      "logps/rejected": -290.7606506347656,
      "loss": 0.284,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.5696079134941101,
      "rewards/margins": 2.2634024620056152,
      "rewards/rejected": -2.833010196685791,
      "step": 150
    },
    {
      "epoch": 0.33066666666666666,
      "grad_norm": 2.194394111633301,
      "learning_rate": 4.918743109018727e-06,
      "logits/chosen": -1.7670875787734985,
      "logits/rejected": -1.5925384759902954,
      "logps/chosen": -198.6534881591797,
      "logps/rejected": -291.90155029296875,
      "loss": 0.1863,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.3875270187854767,
      "rewards/margins": 2.7576119899749756,
      "rewards/rejected": -3.145139217376709,
      "step": 155
    },
    {
      "epoch": 0.3413333333333333,
      "grad_norm": 1.8163162469863892,
      "learning_rate": 4.911302359449585e-06,
      "logits/chosen": -1.767751693725586,
      "logits/rejected": -1.504002571105957,
      "logps/chosen": -224.132568359375,
      "logps/rejected": -301.08026123046875,
      "loss": 0.2234,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.610785722732544,
      "rewards/margins": 2.765779733657837,
      "rewards/rejected": -3.376565456390381,
      "step": 160
    },
    {
      "epoch": 0.352,
      "grad_norm": 2.5315065383911133,
      "learning_rate": 4.9035418250305314e-06,
      "logits/chosen": -1.5424492359161377,
      "logits/rejected": -1.4309223890304565,
      "logps/chosen": -223.1277618408203,
      "logps/rejected": -289.2735290527344,
      "loss": 0.2373,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.6303970217704773,
      "rewards/margins": 2.7104063034057617,
      "rewards/rejected": -3.340803623199463,
      "step": 165
    },
    {
      "epoch": 0.3626666666666667,
      "grad_norm": 2.4730045795440674,
      "learning_rate": 4.895462534956981e-06,
      "logits/chosen": -1.701702356338501,
      "logits/rejected": -1.4543676376342773,
      "logps/chosen": -225.279052734375,
      "logps/rejected": -295.2032470703125,
      "loss": 0.2161,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.5884218215942383,
      "rewards/margins": 2.8115923404693604,
      "rewards/rejected": -3.4000144004821777,
      "step": 170
    },
    {
      "epoch": 0.37333333333333335,
      "grad_norm": 2.8877482414245605,
      "learning_rate": 4.887065560697449e-06,
      "logits/chosen": -1.659183144569397,
      "logits/rejected": -1.4441927671432495,
      "logps/chosen": -220.6221466064453,
      "logps/rejected": -289.29254150390625,
      "loss": 0.2201,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.5501753687858582,
      "rewards/margins": 2.771678924560547,
      "rewards/rejected": -3.3218541145324707,
      "step": 175
    },
    {
      "epoch": 0.384,
      "grad_norm": 2.3038156032562256,
      "learning_rate": 4.878352015851459e-06,
      "logits/chosen": -1.657855749130249,
      "logits/rejected": -1.468774676322937,
      "logps/chosen": -234.57339477539062,
      "logps/rejected": -275.18017578125,
      "loss": 0.2154,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.5347112417221069,
      "rewards/margins": 2.8941256999969482,
      "rewards/rejected": -3.4288368225097656,
      "step": 180
    },
    {
      "epoch": 0.39466666666666667,
      "grad_norm": 1.4912587404251099,
      "learning_rate": 4.869323056001852e-06,
      "logits/chosen": -1.6985185146331787,
      "logits/rejected": -1.4987022876739502,
      "logps/chosen": -215.89541625976562,
      "logps/rejected": -299.5584411621094,
      "loss": 0.2257,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.597873330116272,
      "rewards/margins": 2.936903476715088,
      "rewards/rejected": -3.5347766876220703,
      "step": 185
    },
    {
      "epoch": 0.4053333333333333,
      "grad_norm": 1.758817434310913,
      "learning_rate": 4.859979878561536e-06,
      "logits/chosen": -1.6548064947128296,
      "logits/rejected": -1.480167031288147,
      "logps/chosen": -215.01345825195312,
      "logps/rejected": -302.09722900390625,
      "loss": 0.1969,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7040780186653137,
      "rewards/margins": 2.9713985919952393,
      "rewards/rejected": -3.675476551055908,
      "step": 190
    },
    {
      "epoch": 0.416,
      "grad_norm": 3.2202625274658203,
      "learning_rate": 4.850323722614685e-06,
      "logits/chosen": -1.6637979745864868,
      "logits/rejected": -1.534681797027588,
      "logps/chosen": -223.47787475585938,
      "logps/rejected": -298.9809875488281,
      "loss": 0.2318,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.7906392812728882,
      "rewards/margins": 2.9284045696258545,
      "rewards/rejected": -3.7190442085266113,
      "step": 195
    },
    {
      "epoch": 0.4266666666666667,
      "grad_norm": 3.042365789413452,
      "learning_rate": 4.840355868752416e-06,
      "logits/chosen": -1.6638110876083374,
      "logits/rejected": -1.4790096282958984,
      "logps/chosen": -259.5401306152344,
      "logps/rejected": -279.5474853515625,
      "loss": 0.2538,
      "rewards/accuracies": 0.8812500238418579,
      "rewards/chosen": -0.3952628970146179,
      "rewards/margins": 2.784644603729248,
      "rewards/rejected": -3.179908037185669,
      "step": 200
    },
    {
      "epoch": 0.43733333333333335,
      "grad_norm": 3.7375149726867676,
      "learning_rate": 4.830077638902955e-06,
      "logits/chosen": -1.772063970565796,
      "logits/rejected": -1.5635404586791992,
      "logps/chosen": -192.68406677246094,
      "logps/rejected": -279.232421875,
      "loss": 0.2313,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6237959861755371,
      "rewards/margins": 3.2594666481018066,
      "rewards/rejected": -3.8832626342773438,
      "step": 205
    },
    {
      "epoch": 0.448,
      "grad_norm": 2.7821362018585205,
      "learning_rate": 4.8194903961563255e-06,
      "logits/chosen": -1.8043403625488281,
      "logits/rejected": -1.5912580490112305,
      "logps/chosen": -212.60244750976562,
      "logps/rejected": -303.67816162109375,
      "loss": 0.176,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.5918294191360474,
      "rewards/margins": 3.186065196990967,
      "rewards/rejected": -3.7778944969177246,
      "step": 210
    },
    {
      "epoch": 0.45866666666666667,
      "grad_norm": 3.463007926940918,
      "learning_rate": 4.808595544583578e-06,
      "logits/chosen": -1.763313889503479,
      "logits/rejected": -1.6367692947387695,
      "logps/chosen": -202.7022705078125,
      "logps/rejected": -306.5892028808594,
      "loss": 0.2357,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.530913233757019,
      "rewards/margins": 3.3871676921844482,
      "rewards/rejected": -3.9180808067321777,
      "step": 215
    },
    {
      "epoch": 0.4693333333333333,
      "grad_norm": 2.2896623611450195,
      "learning_rate": 4.797394529050577e-06,
      "logits/chosen": -1.6672195196151733,
      "logits/rejected": -1.4932286739349365,
      "logps/chosen": -249.15420532226562,
      "logps/rejected": -311.45587158203125,
      "loss": 0.1972,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7217354774475098,
      "rewards/margins": 3.4966182708740234,
      "rewards/rejected": -4.218353271484375,
      "step": 220
    },
    {
      "epoch": 0.48,
      "grad_norm": 1.703731656074524,
      "learning_rate": 4.785888835026393e-06,
      "logits/chosen": -1.7073551416397095,
      "logits/rejected": -1.517589807510376,
      "logps/chosen": -207.189208984375,
      "logps/rejected": -306.99774169921875,
      "loss": 0.1687,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.24462179839611053,
      "rewards/margins": 3.6275787353515625,
      "rewards/rejected": -3.8722004890441895,
      "step": 225
    },
    {
      "epoch": 0.49066666666666664,
      "grad_norm": 2.8105475902557373,
      "learning_rate": 4.7740799883862966e-06,
      "logits/chosen": -1.708707571029663,
      "logits/rejected": -1.5929806232452393,
      "logps/chosen": -206.3466339111328,
      "logps/rejected": -301.8596496582031,
      "loss": 0.1369,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.34434959292411804,
      "rewards/margins": 3.6625962257385254,
      "rewards/rejected": -4.006945610046387,
      "step": 230
    },
    {
      "epoch": 0.5013333333333333,
      "grad_norm": 3.4738473892211914,
      "learning_rate": 4.761969555209394e-06,
      "logits/chosen": -1.7533724308013916,
      "logits/rejected": -1.5469913482666016,
      "logps/chosen": -203.04383850097656,
      "logps/rejected": -297.419921875,
      "loss": 0.1751,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4458187520503998,
      "rewards/margins": 4.078585624694824,
      "rewards/rejected": -4.524405002593994,
      "step": 235
    },
    {
      "epoch": 0.512,
      "grad_norm": 1.7620586156845093,
      "learning_rate": 4.7495591415709435e-06,
      "logits/chosen": -1.6817264556884766,
      "logits/rejected": -1.5172640085220337,
      "logps/chosen": -204.63270568847656,
      "logps/rejected": -305.5479431152344,
      "loss": 0.1515,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.46541064977645874,
      "rewards/margins": 4.120365142822266,
      "rewards/rejected": -4.585775852203369,
      "step": 240
    },
    {
      "epoch": 0.5226666666666666,
      "grad_norm": 1.4158145189285278,
      "learning_rate": 4.736850393329353e-06,
      "logits/chosen": -1.752340316772461,
      "logits/rejected": -1.503805160522461,
      "logps/chosen": -224.72238159179688,
      "logps/rejected": -302.3330078125,
      "loss": 0.2077,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.7848609685897827,
      "rewards/margins": 3.807910919189453,
      "rewards/rejected": -4.592772483825684,
      "step": 245
    },
    {
      "epoch": 0.5333333333333333,
      "grad_norm": 2.476516008377075,
      "learning_rate": 4.7238449959079095e-06,
      "logits/chosen": -1.7871164083480835,
      "logits/rejected": -1.5389041900634766,
      "logps/chosen": -208.1581268310547,
      "logps/rejected": -295.5426330566406,
      "loss": 0.1566,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7614593505859375,
      "rewards/margins": 4.342654228210449,
      "rewards/rejected": -5.104113578796387,
      "step": 250
    },
    {
      "epoch": 0.544,
      "grad_norm": 2.2022173404693604,
      "learning_rate": 4.710544674071259e-06,
      "logits/chosen": -1.7751401662826538,
      "logits/rejected": -1.5775682926177979,
      "logps/chosen": -266.8836669921875,
      "logps/rejected": -283.87957763671875,
      "loss": 0.2096,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.43025732040405273,
      "rewards/margins": 4.187695503234863,
      "rewards/rejected": -4.617953300476074,
      "step": 255
    },
    {
      "epoch": 0.5546666666666666,
      "grad_norm": 6.2148942947387695,
      "learning_rate": 4.696951191696673e-06,
      "logits/chosen": -1.8162314891815186,
      "logits/rejected": -1.5676801204681396,
      "logps/chosen": -198.07168579101562,
      "logps/rejected": -300.4014892578125,
      "loss": 0.2171,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.8536887168884277,
      "rewards/margins": 4.168699741363525,
      "rewards/rejected": -5.022388458251953,
      "step": 260
    },
    {
      "epoch": 0.5653333333333334,
      "grad_norm": 2.747276782989502,
      "learning_rate": 4.6830663515401196e-06,
      "logits/chosen": -1.810840368270874,
      "logits/rejected": -1.5849469900131226,
      "logps/chosen": -223.6453857421875,
      "logps/rejected": -289.7298278808594,
      "loss": 0.1779,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.4079108238220215,
      "rewards/margins": 4.084084987640381,
      "rewards/rejected": -4.491995811462402,
      "step": 265
    },
    {
      "epoch": 0.576,
      "grad_norm": 2.140824794769287,
      "learning_rate": 4.6688919949971855e-06,
      "logits/chosen": -1.7053123712539673,
      "logits/rejected": -1.4852755069732666,
      "logps/chosen": -224.3308563232422,
      "logps/rejected": -324.8773498535156,
      "loss": 0.1714,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.6945469975471497,
      "rewards/margins": 4.143045902252197,
      "rewards/rejected": -4.837592601776123,
      "step": 270
    },
    {
      "epoch": 0.5866666666666667,
      "grad_norm": 1.720471739768982,
      "learning_rate": 4.654430001858874e-06,
      "logits/chosen": -1.5651119947433472,
      "logits/rejected": -1.4575681686401367,
      "logps/chosen": -230.16384887695312,
      "logps/rejected": -316.0377197265625,
      "loss": 0.1983,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.7454553842544556,
      "rewards/margins": 3.871629238128662,
      "rewards/rejected": -4.617084980010986,
      "step": 275
    },
    {
      "epoch": 0.5973333333333334,
      "grad_norm": 1.851474404335022,
      "learning_rate": 4.6396822900623075e-06,
      "logits/chosen": -1.7224136590957642,
      "logits/rejected": -1.5002939701080322,
      "logps/chosen": -214.7126922607422,
      "logps/rejected": -302.85430908203125,
      "loss": 0.1892,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.7812764644622803,
      "rewards/margins": 4.019623756408691,
      "rewards/rejected": -4.800899982452393,
      "step": 280
    },
    {
      "epoch": 0.608,
      "grad_norm": 2.1525115966796875,
      "learning_rate": 4.624650815436368e-06,
      "logits/chosen": -1.7956489324569702,
      "logits/rejected": -1.5618733167648315,
      "logps/chosen": -240.7410430908203,
      "logps/rejected": -317.7781982421875,
      "loss": 0.1822,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -0.6186224222183228,
      "rewards/margins": 4.1468329429626465,
      "rewards/rejected": -4.765455722808838,
      "step": 285
    },
    {
      "epoch": 0.6186666666666667,
      "grad_norm": 1.5195742845535278,
      "learning_rate": 4.609337571442324e-06,
      "logits/chosen": -1.8218319416046143,
      "logits/rejected": -1.6440036296844482,
      "logps/chosen": -199.41769409179688,
      "logps/rejected": -308.4180908203125,
      "loss": 0.1525,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6784080266952515,
      "rewards/margins": 4.462456703186035,
      "rewards/rejected": -5.140865325927734,
      "step": 290
    },
    {
      "epoch": 0.6293333333333333,
      "grad_norm": 2.91068172454834,
      "learning_rate": 4.5937445889094535e-06,
      "logits/chosen": -1.7051767110824585,
      "logits/rejected": -1.537472128868103,
      "logps/chosen": -234.99716186523438,
      "logps/rejected": -309.03814697265625,
      "loss": 0.2014,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.6108633279800415,
      "rewards/margins": 3.835562229156494,
      "rewards/rejected": -4.446425437927246,
      "step": 295
    },
    {
      "epoch": 0.64,
      "grad_norm": 0.7544635534286499,
      "learning_rate": 4.577873935765722e-06,
      "logits/chosen": -1.8061891794204712,
      "logits/rejected": -1.581192970275879,
      "logps/chosen": -193.08706665039062,
      "logps/rejected": -302.5655517578125,
      "loss": 0.1642,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.450061172246933,
      "rewards/margins": 4.481039524078369,
      "rewards/rejected": -4.931101322174072,
      "step": 300
    },
    {
      "epoch": 0.6506666666666666,
      "grad_norm": 3.741976022720337,
      "learning_rate": 4.561727716763532e-06,
      "logits/chosen": -1.750154733657837,
      "logits/rejected": -1.6348270177841187,
      "logps/chosen": -232.0447235107422,
      "logps/rejected": -299.55377197265625,
      "loss": 0.2224,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.28944095969200134,
      "rewards/margins": 4.048866271972656,
      "rewards/rejected": -4.3383073806762695,
      "step": 305
    },
    {
      "epoch": 0.6613333333333333,
      "grad_norm": 2.625056028366089,
      "learning_rate": 4.545308073200591e-06,
      "logits/chosen": -1.7072067260742188,
      "logits/rejected": -1.515326738357544,
      "logps/chosen": -219.8765106201172,
      "logps/rejected": -305.98492431640625,
      "loss": 0.1724,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.3121286928653717,
      "rewards/margins": 4.423548698425293,
      "rewards/rejected": -4.7356767654418945,
      "step": 310
    },
    {
      "epoch": 0.672,
      "grad_norm": 3.3447206020355225,
      "learning_rate": 4.528617182635943e-06,
      "logits/chosen": -1.7671434879302979,
      "logits/rejected": -1.5995731353759766,
      "logps/chosen": -213.7782745361328,
      "logps/rejected": -303.6085205078125,
      "loss": 0.2261,
      "rewards/accuracies": 0.90625,
      "rewards/chosen": -0.4999776780605316,
      "rewards/margins": 4.172996520996094,
      "rewards/rejected": -4.672974586486816,
      "step": 315
    },
    {
      "epoch": 0.6826666666666666,
      "grad_norm": 3.0568952560424805,
      "learning_rate": 4.511657258601171e-06,
      "logits/chosen": -1.7790120840072632,
      "logits/rejected": -1.5551685094833374,
      "logps/chosen": -219.55514526367188,
      "logps/rejected": -298.2545471191406,
      "loss": 0.1924,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.366494745016098,
      "rewards/margins": 4.068981647491455,
      "rewards/rejected": -4.435476303100586,
      "step": 320
    },
    {
      "epoch": 0.6933333333333334,
      "grad_norm": 2.1040990352630615,
      "learning_rate": 4.494430550306848e-06,
      "logits/chosen": -1.5438107252120972,
      "logits/rejected": -1.4709008932113647,
      "logps/chosen": -226.4731903076172,
      "logps/rejected": -295.2748718261719,
      "loss": 0.1854,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.3337394595146179,
      "rewards/margins": 3.975902557373047,
      "rewards/rejected": -4.3096418380737305,
      "step": 325
    },
    {
      "epoch": 0.704,
      "grad_norm": 2.9654102325439453,
      "learning_rate": 4.476939342344246e-06,
      "logits/chosen": -1.794424057006836,
      "logits/rejected": -1.5975098609924316,
      "logps/chosen": -220.2859344482422,
      "logps/rejected": -298.57977294921875,
      "loss": 0.1412,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.4857562482357025,
      "rewards/margins": 4.390166759490967,
      "rewards/rejected": -4.875923156738281,
      "step": 330
    },
    {
      "epoch": 0.7146666666666667,
      "grad_norm": 2.9396848678588867,
      "learning_rate": 4.45918595438236e-06,
      "logits/chosen": -1.595558762550354,
      "logits/rejected": -1.5090906620025635,
      "logps/chosen": -215.38565063476562,
      "logps/rejected": -291.2980041503906,
      "loss": 0.1736,
      "rewards/accuracies": 0.9125000238418579,
      "rewards/chosen": -0.33694586157798767,
      "rewards/margins": 4.20229434967041,
      "rewards/rejected": -4.53924036026001,
      "step": 335
    },
    {
      "epoch": 0.7253333333333334,
      "grad_norm": 2.5800886154174805,
      "learning_rate": 4.441172740860267e-06,
      "logits/chosen": -1.6961015462875366,
      "logits/rejected": -1.5860854387283325,
      "logps/chosen": -208.26205444335938,
      "logps/rejected": -289.4010314941406,
      "loss": 0.2338,
      "rewards/accuracies": 0.918749988079071,
      "rewards/chosen": -0.3276976943016052,
      "rewards/margins": 3.7353787422180176,
      "rewards/rejected": -4.063076019287109,
      "step": 340
    },
    {
      "epoch": 0.736,
      "grad_norm": 1.9638211727142334,
      "learning_rate": 4.422902090674886e-06,
      "logits/chosen": -1.658978819847107,
      "logits/rejected": -1.5040454864501953,
      "logps/chosen": -203.96173095703125,
      "logps/rejected": -306.2127990722656,
      "loss": 0.1547,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.2021322250366211,
      "rewards/margins": 4.683990955352783,
      "rewards/rejected": -4.886122703552246,
      "step": 345
    },
    {
      "epoch": 0.7466666666666667,
      "grad_norm": 1.643659234046936,
      "learning_rate": 4.4043764268641705e-06,
      "logits/chosen": -1.7135502099990845,
      "logits/rejected": -1.5774753093719482,
      "logps/chosen": -214.1969451904297,
      "logps/rejected": -296.029541015625,
      "loss": 0.1529,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.4376668930053711,
      "rewards/margins": 3.894376039505005,
      "rewards/rejected": -4.332043170928955,
      "step": 350
    },
    {
      "epoch": 0.7573333333333333,
      "grad_norm": 1.9113845825195312,
      "learning_rate": 4.385598206285758e-06,
      "logits/chosen": -1.7747424840927124,
      "logits/rejected": -1.6252682209014893,
      "logps/chosen": -212.4604034423828,
      "logps/rejected": -336.4494323730469,
      "loss": 0.1163,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -0.5154281258583069,
      "rewards/margins": 5.046199321746826,
      "rewards/rejected": -5.561627388000488,
      "step": 355
    },
    {
      "epoch": 0.768,
      "grad_norm": 2.6899571418762207,
      "learning_rate": 4.3665699192911495e-06,
      "logits/chosen": -1.6771185398101807,
      "logits/rejected": -1.5176887512207031,
      "logps/chosen": -204.26272583007812,
      "logps/rejected": -299.89288330078125,
      "loss": 0.1435,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.373568058013916,
      "rewards/margins": 4.497839450836182,
      "rewards/rejected": -4.871407508850098,
      "step": 360
    },
    {
      "epoch": 0.7786666666666666,
      "grad_norm": 2.5716707706451416,
      "learning_rate": 4.347294089395438e-06,
      "logits/chosen": -1.7271244525909424,
      "logits/rejected": -1.5585415363311768,
      "logps/chosen": -234.6363525390625,
      "logps/rejected": -342.66668701171875,
      "loss": 0.1127,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.6868476271629333,
      "rewards/margins": 5.073822975158691,
      "rewards/rejected": -5.7606706619262695,
      "step": 365
    },
    {
      "epoch": 0.7893333333333333,
      "grad_norm": 1.3837978839874268,
      "learning_rate": 4.327773272942647e-06,
      "logits/chosen": -1.6469981670379639,
      "logits/rejected": -1.5777738094329834,
      "logps/chosen": -203.450439453125,
      "logps/rejected": -331.1883544921875,
      "loss": 0.154,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.6987647414207458,
      "rewards/margins": 4.91748046875,
      "rewards/rejected": -5.616245746612549,
      "step": 370
    },
    {
      "epoch": 0.8,
      "grad_norm": 2.062950849533081,
      "learning_rate": 4.308010058766703e-06,
      "logits/chosen": -1.829921007156372,
      "logits/rejected": -1.6271889209747314,
      "logps/chosen": -229.3686981201172,
      "logps/rejected": -320.89117431640625,
      "loss": 0.1308,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9174984693527222,
      "rewards/margins": 5.067828178405762,
      "rewards/rejected": -5.985327243804932,
      "step": 375
    },
    {
      "epoch": 0.8106666666666666,
      "grad_norm": 2.347057580947876,
      "learning_rate": 4.2880070678481135e-06,
      "logits/chosen": -1.7627887725830078,
      "logits/rejected": -1.5318098068237305,
      "logps/chosen": -199.64096069335938,
      "logps/rejected": -312.7142028808594,
      "loss": 0.1086,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.8036954998970032,
      "rewards/margins": 5.584313869476318,
      "rewards/rejected": -6.388009071350098,
      "step": 380
    },
    {
      "epoch": 0.8213333333333334,
      "grad_norm": 3.017650604248047,
      "learning_rate": 4.267766952966369e-06,
      "logits/chosen": -1.7800073623657227,
      "logits/rejected": -1.585495948791504,
      "logps/chosen": -227.3597412109375,
      "logps/rejected": -320.13653564453125,
      "loss": 0.1488,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9181621670722961,
      "rewards/margins": 5.3811445236206055,
      "rewards/rejected": -6.299307346343994,
      "step": 385
    },
    {
      "epoch": 0.832,
      "grad_norm": 3.2862038612365723,
      "learning_rate": 4.24729239834814e-06,
      "logits/chosen": -1.7047274112701416,
      "logits/rejected": -1.5511637926101685,
      "logps/chosen": -229.1103057861328,
      "logps/rejected": -305.8432312011719,
      "loss": 0.1743,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.0054670572280884,
      "rewards/margins": 4.895895957946777,
      "rewards/rejected": -5.901362895965576,
      "step": 390
    },
    {
      "epoch": 0.8426666666666667,
      "grad_norm": 2.632439136505127,
      "learning_rate": 4.226586119311291e-06,
      "logits/chosen": -1.7877633571624756,
      "logits/rejected": -1.6740882396697998,
      "logps/chosen": -217.3224334716797,
      "logps/rejected": -324.86737060546875,
      "loss": 0.155,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9102714657783508,
      "rewards/margins": 5.167611598968506,
      "rewards/rejected": -6.077882766723633,
      "step": 395
    },
    {
      "epoch": 0.8533333333333334,
      "grad_norm": 3.4450037479400635,
      "learning_rate": 4.205650861904782e-06,
      "logits/chosen": -1.7734384536743164,
      "logits/rejected": -1.5797088146209717,
      "logps/chosen": -212.7061309814453,
      "logps/rejected": -316.12127685546875,
      "loss": 0.1384,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.066833734512329,
      "rewards/margins": 5.305598258972168,
      "rewards/rejected": -6.372431755065918,
      "step": 400
    },
    {
      "epoch": 0.864,
      "grad_norm": 0.9102749228477478,
      "learning_rate": 4.184489402544486e-06,
      "logits/chosen": -1.820951223373413,
      "logits/rejected": -1.6782753467559814,
      "logps/chosen": -207.24374389648438,
      "logps/rejected": -337.61700439453125,
      "loss": 0.1447,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.7356926202774048,
      "rewards/margins": 5.745200157165527,
      "rewards/rejected": -6.480892181396484,
      "step": 405
    },
    {
      "epoch": 0.8746666666666667,
      "grad_norm": 2.6713948249816895,
      "learning_rate": 4.1631045476449865e-06,
      "logits/chosen": -1.8452482223510742,
      "logits/rejected": -1.6467907428741455,
      "logps/chosen": -204.97213745117188,
      "logps/rejected": -308.3583984375,
      "loss": 0.2012,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.972851574420929,
      "rewards/margins": 4.9895501136779785,
      "rewards/rejected": -5.962401866912842,
      "step": 410
    },
    {
      "epoch": 0.8853333333333333,
      "grad_norm": 1.5577032566070557,
      "learning_rate": 4.141499133247391e-06,
      "logits/chosen": -1.6295883655548096,
      "logits/rejected": -1.6040502786636353,
      "logps/chosen": -216.0922393798828,
      "logps/rejected": -305.439208984375,
      "loss": 0.1682,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.7497788667678833,
      "rewards/margins": 4.673377990722656,
      "rewards/rejected": -5.423156261444092,
      "step": 415
    },
    {
      "epoch": 0.896,
      "grad_norm": 2.909951686859131,
      "learning_rate": 4.119676024643217e-06,
      "logits/chosen": -1.7234691381454468,
      "logits/rejected": -1.5772818326950073,
      "logps/chosen": -221.32339477539062,
      "logps/rejected": -318.9827880859375,
      "loss": 0.1345,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -0.6560640335083008,
      "rewards/margins": 5.7348504066467285,
      "rewards/rejected": -6.390913963317871,
      "step": 420
    },
    {
      "epoch": 0.9066666666666666,
      "grad_norm": 4.880727291107178,
      "learning_rate": 4.0976381159944055e-06,
      "logits/chosen": -1.7095870971679688,
      "logits/rejected": -1.5015006065368652,
      "logps/chosen": -239.50650024414062,
      "logps/rejected": -340.181884765625,
      "loss": 0.1615,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -0.8043699264526367,
      "rewards/margins": 5.589447975158691,
      "rewards/rejected": -6.393817901611328,
      "step": 425
    },
    {
      "epoch": 0.9173333333333333,
      "grad_norm": 1.4485254287719727,
      "learning_rate": 4.07538832994949e-06,
      "logits/chosen": -1.7408502101898193,
      "logits/rejected": -1.5344233512878418,
      "logps/chosen": -217.7578582763672,
      "logps/rejected": -321.7436218261719,
      "loss": 0.1404,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.0381317138671875,
      "rewards/margins": 5.289247512817383,
      "rewards/rejected": -6.3273797035217285,
      "step": 430
    },
    {
      "epoch": 0.928,
      "grad_norm": 2.7532310485839844,
      "learning_rate": 4.0529296172560055e-06,
      "logits/chosen": -1.829403281211853,
      "logits/rejected": -1.6238079071044922,
      "logps/chosen": -212.47366333007812,
      "logps/rejected": -326.38494873046875,
      "loss": 0.1214,
      "rewards/accuracies": 0.925000011920929,
      "rewards/chosen": -1.2220343351364136,
      "rewards/margins": 5.693434715270996,
      "rewards/rejected": -6.915470123291016,
      "step": 435
    },
    {
      "epoch": 0.9386666666666666,
      "grad_norm": 2.052518844604492,
      "learning_rate": 4.030264956369158e-06,
      "logits/chosen": -1.7937902212142944,
      "logits/rejected": -1.673724889755249,
      "logps/chosen": -213.76602172851562,
      "logps/rejected": -336.3763732910156,
      "loss": 0.1351,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -0.9977391958236694,
      "rewards/margins": 5.939627170562744,
      "rewards/rejected": -6.9373674392700195,
      "step": 440
    },
    {
      "epoch": 0.9493333333333334,
      "grad_norm": 5.171205043792725,
      "learning_rate": 4.0073973530568274e-06,
      "logits/chosen": -1.762075424194336,
      "logits/rejected": -1.5835630893707275,
      "logps/chosen": -238.7696990966797,
      "logps/rejected": -327.5137634277344,
      "loss": 0.153,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.2305481433868408,
      "rewards/margins": 5.5450639724731445,
      "rewards/rejected": -6.77561092376709,
      "step": 445
    },
    {
      "epoch": 0.96,
      "grad_norm": 1.4903888702392578,
      "learning_rate": 3.9843298400009435e-06,
      "logits/chosen": -1.7811949253082275,
      "logits/rejected": -1.6663089990615845,
      "logps/chosen": -197.07852172851562,
      "logps/rejected": -334.981201171875,
      "loss": 0.0804,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.1236552000045776,
      "rewards/margins": 6.174225330352783,
      "rewards/rejected": -7.29788064956665,
      "step": 450
    },
    {
      "epoch": 0.9706666666666667,
      "grad_norm": 3.032621145248413,
      "learning_rate": 3.961065476395296e-06,
      "logits/chosen": -1.8615983724594116,
      "logits/rejected": -1.6502336263656616,
      "logps/chosen": -223.0669403076172,
      "logps/rejected": -337.3778991699219,
      "loss": 0.1113,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.9720472097396851,
      "rewards/margins": 6.413779258728027,
      "rewards/rejected": -7.385826110839844,
      "step": 455
    },
    {
      "epoch": 0.9813333333333333,
      "grad_norm": 2.9131219387054443,
      "learning_rate": 3.937607347539823e-06,
      "logits/chosen": -1.839909553527832,
      "logits/rejected": -1.635088562965393,
      "logps/chosen": -208.74105834960938,
      "logps/rejected": -329.6622619628906,
      "loss": 0.17,
      "rewards/accuracies": 0.9312499761581421,
      "rewards/chosen": -1.350288987159729,
      "rewards/margins": 6.138724327087402,
      "rewards/rejected": -7.489013671875,
      "step": 460
    },
    {
      "epoch": 0.992,
      "grad_norm": 1.870213508605957,
      "learning_rate": 3.913958564431447e-06,
      "logits/chosen": -1.8265674114227295,
      "logits/rejected": -1.642881155014038,
      "logps/chosen": -230.4730682373047,
      "logps/rejected": -343.09173583984375,
      "loss": 0.1161,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3268934488296509,
      "rewards/margins": 6.083160877227783,
      "rewards/rejected": -7.410054683685303,
      "step": 465
    },
    {
      "epoch": 1.0021333333333333,
      "grad_norm": 3.683117151260376,
      "learning_rate": 3.89012226335149e-06,
      "logits/chosen": -1.7430078983306885,
      "logits/rejected": -1.6155897378921509,
      "logps/chosen": -204.52291870117188,
      "logps/rejected": -320.3079833984375,
      "loss": 0.1491,
      "rewards/accuracies": 0.9276315569877625,
      "rewards/chosen": -1.002981424331665,
      "rewards/margins": 5.943866729736328,
      "rewards/rejected": -6.946847915649414,
      "step": 470
    },
    {
      "epoch": 1.0128,
      "grad_norm": 1.268686294555664,
      "learning_rate": 3.86610160544975e-06,
      "logits/chosen": -1.8006131649017334,
      "logits/rejected": -1.650631308555603,
      "logps/chosen": -208.288818359375,
      "logps/rejected": -330.7633972167969,
      "loss": 0.1072,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.0482348203659058,
      "rewards/margins": 6.511569023132324,
      "rewards/rejected": -7.5598039627075195,
      "step": 475
    },
    {
      "epoch": 1.0234666666666667,
      "grad_norm": 1.9249908924102783,
      "learning_rate": 3.841899776325267e-06,
      "logits/chosen": -1.7247737646102905,
      "logits/rejected": -1.6122996807098389,
      "logps/chosen": -229.8289337158203,
      "logps/rejected": -347.37017822265625,
      "loss": 0.1015,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -0.8600400686264038,
      "rewards/margins": 6.6198883056640625,
      "rewards/rejected": -7.479928493499756,
      "step": 480
    },
    {
      "epoch": 1.0341333333333333,
      "grad_norm": 1.7193055152893066,
      "learning_rate": 3.817519985603855e-06,
      "logits/chosen": -1.8941304683685303,
      "logits/rejected": -1.7352015972137451,
      "logps/chosen": -212.7666778564453,
      "logps/rejected": -335.093994140625,
      "loss": 0.1174,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.8410255908966064,
      "rewards/margins": 6.038805961608887,
      "rewards/rejected": -7.8798322677612305,
      "step": 485
    },
    {
      "epoch": 1.0448,
      "grad_norm": 3.5899593830108643,
      "learning_rate": 3.792965466512437e-06,
      "logits/chosen": -1.7666019201278687,
      "logits/rejected": -1.573142409324646,
      "logps/chosen": -232.1917266845703,
      "logps/rejected": -326.2380065917969,
      "loss": 0.1254,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2813751697540283,
      "rewards/margins": 6.1387457847595215,
      "rewards/rejected": -7.4201202392578125,
      "step": 490
    },
    {
      "epoch": 1.0554666666666668,
      "grad_norm": 3.8329415321350098,
      "learning_rate": 3.7682394754502687e-06,
      "logits/chosen": -1.7883208990097046,
      "logits/rejected": -1.6514673233032227,
      "logps/chosen": -220.38290405273438,
      "logps/rejected": -339.58355712890625,
      "loss": 0.1327,
      "rewards/accuracies": 0.9437500238418579,
      "rewards/chosen": -1.6085094213485718,
      "rewards/margins": 6.206122398376465,
      "rewards/rejected": -7.814632415771484,
      "step": 495
    },
    {
      "epoch": 1.0661333333333334,
      "grad_norm": 2.348676919937134,
      "learning_rate": 3.7433452915570685e-06,
      "logits/chosen": -1.8436359167099,
      "logits/rejected": -1.6850477457046509,
      "logps/chosen": -236.15066528320312,
      "logps/rejected": -352.1568908691406,
      "loss": 0.0809,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5462639331817627,
      "rewards/margins": 6.4064435958862305,
      "rewards/rejected": -7.952707767486572,
      "step": 500
    },
    {
      "epoch": 1.0768,
      "grad_norm": 0.5271938443183899,
      "learning_rate": 3.718286216278143e-06,
      "logits/chosen": -1.7932989597320557,
      "logits/rejected": -1.6435298919677734,
      "logps/chosen": -217.5049591064453,
      "logps/rejected": -361.5685729980469,
      "loss": 0.0622,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.248918056488037,
      "rewards/margins": 7.289629936218262,
      "rewards/rejected": -8.538548469543457,
      "step": 505
    },
    {
      "epoch": 1.0874666666666666,
      "grad_norm": 2.5198798179626465,
      "learning_rate": 3.6930655729265557e-06,
      "logits/chosen": -1.8247435092926025,
      "logits/rejected": -1.648455023765564,
      "logps/chosen": -252.8979949951172,
      "logps/rejected": -347.34930419921875,
      "loss": 0.1646,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7946258783340454,
      "rewards/margins": 6.223298072814941,
      "rewards/rejected": -8.017923355102539,
      "step": 510
    },
    {
      "epoch": 1.0981333333333334,
      "grad_norm": 2.4615252017974854,
      "learning_rate": 3.667686706242386e-06,
      "logits/chosen": -1.748671293258667,
      "logits/rejected": -1.6229207515716553,
      "logps/chosen": -230.9926300048828,
      "logps/rejected": -341.594970703125,
      "loss": 0.1012,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.5733277797698975,
      "rewards/margins": 6.404208183288574,
      "rewards/rejected": -7.977537631988525,
      "step": 515
    },
    {
      "epoch": 1.1088,
      "grad_norm": 1.5133553743362427,
      "learning_rate": 3.6421529819491576e-06,
      "logits/chosen": -1.8420203924179077,
      "logits/rejected": -1.71492600440979,
      "logps/chosen": -239.5989532470703,
      "logps/rejected": -357.04937744140625,
      "loss": 0.075,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.4472997188568115,
      "rewards/margins": 7.079667091369629,
      "rewards/rejected": -8.526968002319336,
      "step": 520
    },
    {
      "epoch": 1.1194666666666666,
      "grad_norm": 1.1103506088256836,
      "learning_rate": 3.6164677863074783e-06,
      "logits/chosen": -1.7879250049591064,
      "logits/rejected": -1.6893844604492188,
      "logps/chosen": -215.9795684814453,
      "logps/rejected": -359.2721252441406,
      "loss": 0.0696,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.6059297323226929,
      "rewards/margins": 7.181890964508057,
      "rewards/rejected": -8.787820816040039,
      "step": 525
    },
    {
      "epoch": 1.1301333333333332,
      "grad_norm": 1.1953598260879517,
      "learning_rate": 3.590634525665955e-06,
      "logits/chosen": -1.8406591415405273,
      "logits/rejected": -1.6219384670257568,
      "logps/chosen": -224.5021514892578,
      "logps/rejected": -349.89239501953125,
      "loss": 0.0765,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9252121448516846,
      "rewards/margins": 6.995842933654785,
      "rewards/rejected": -8.921055793762207,
      "step": 530
    },
    {
      "epoch": 1.1408,
      "grad_norm": 1.1147931814193726,
      "learning_rate": 3.564656626009446e-06,
      "logits/chosen": -1.8823022842407227,
      "logits/rejected": -1.8116251230239868,
      "logps/chosen": -210.24227905273438,
      "logps/rejected": -362.93145751953125,
      "loss": 0.1049,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.5180342197418213,
      "rewards/margins": 7.260135650634766,
      "rewards/rejected": -8.778169631958008,
      "step": 535
    },
    {
      "epoch": 1.1514666666666666,
      "grad_norm": 2.6945362091064453,
      "learning_rate": 3.5385375325047167e-06,
      "logits/chosen": -1.9016218185424805,
      "logits/rejected": -1.729413628578186,
      "logps/chosen": -270.2596130371094,
      "logps/rejected": -332.03448486328125,
      "loss": 0.0759,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.4495543241500854,
      "rewards/margins": 6.9950737953186035,
      "rewards/rejected": -8.44462776184082,
      "step": 540
    },
    {
      "epoch": 1.1621333333333332,
      "grad_norm": 3.8907995223999023,
      "learning_rate": 3.512280709043534e-06,
      "logits/chosen": -1.6742641925811768,
      "logits/rejected": -1.541598916053772,
      "logps/chosen": -226.13583374023438,
      "logps/rejected": -314.2048034667969,
      "loss": 0.1446,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6109225749969482,
      "rewards/margins": 5.946639060974121,
      "rewards/rejected": -7.55756139755249,
      "step": 545
    },
    {
      "epoch": 1.1728,
      "grad_norm": 5.299363613128662,
      "learning_rate": 3.4858896377832966e-06,
      "logits/chosen": -1.779280424118042,
      "logits/rejected": -1.6656166315078735,
      "logps/chosen": -223.9244384765625,
      "logps/rejected": -348.4911193847656,
      "loss": 0.1078,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9482396841049194,
      "rewards/margins": 6.998904228210449,
      "rewards/rejected": -8.947145462036133,
      "step": 550
    },
    {
      "epoch": 1.1834666666666667,
      "grad_norm": 3.3822855949401855,
      "learning_rate": 3.459367818685231e-06,
      "logits/chosen": -1.7240779399871826,
      "logits/rejected": -1.656288504600525,
      "logps/chosen": -236.51571655273438,
      "logps/rejected": -341.683837890625,
      "loss": 0.1403,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5938842296600342,
      "rewards/margins": 6.741963863372803,
      "rewards/rejected": -8.335847854614258,
      "step": 555
    },
    {
      "epoch": 1.1941333333333333,
      "grad_norm": 0.8533849716186523,
      "learning_rate": 3.432718769050228e-06,
      "logits/chosen": -1.6910005807876587,
      "logits/rejected": -1.5575242042541504,
      "logps/chosen": -234.0883026123047,
      "logps/rejected": -338.9890441894531,
      "loss": 0.1177,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.399706244468689,
      "rewards/margins": 6.219531059265137,
      "rewards/rejected": -7.619238376617432,
      "step": 560
    },
    {
      "epoch": 1.2048,
      "grad_norm": 1.684748888015747,
      "learning_rate": 3.405946023052386e-06,
      "logits/chosen": -1.811579942703247,
      "logits/rejected": -1.6530382633209229,
      "logps/chosen": -210.05844116210938,
      "logps/rejected": -330.2632141113281,
      "loss": 0.0964,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.36553955078125,
      "rewards/margins": 6.456725120544434,
      "rewards/rejected": -7.822264671325684,
      "step": 565
    },
    {
      "epoch": 1.2154666666666667,
      "grad_norm": 4.0320587158203125,
      "learning_rate": 3.3790531312703055e-06,
      "logits/chosen": -1.7837402820587158,
      "logits/rejected": -1.6148713827133179,
      "logps/chosen": -215.8035125732422,
      "logps/rejected": -337.56805419921875,
      "loss": 0.0946,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.377327561378479,
      "rewards/margins": 6.664053440093994,
      "rewards/rejected": -8.041379928588867,
      "step": 570
    },
    {
      "epoch": 1.2261333333333333,
      "grad_norm": 2.986213445663452,
      "learning_rate": 3.35204366021622e-06,
      "logits/chosen": -1.7007958889007568,
      "logits/rejected": -1.554639220237732,
      "logps/chosen": -231.6057891845703,
      "logps/rejected": -361.81781005859375,
      "loss": 0.1045,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2639760971069336,
      "rewards/margins": 6.448613166809082,
      "rewards/rejected": -7.712589263916016,
      "step": 575
    },
    {
      "epoch": 1.2368000000000001,
      "grad_norm": 1.3645539283752441,
      "learning_rate": 3.3249211918630055e-06,
      "logits/chosen": -1.7643219232559204,
      "logits/rejected": -1.6710097789764404,
      "logps/chosen": -210.04476928710938,
      "logps/rejected": -327.1766662597656,
      "loss": 0.1177,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.2976925373077393,
      "rewards/margins": 5.873549461364746,
      "rewards/rejected": -7.171242713928223,
      "step": 580
    },
    {
      "epoch": 1.2474666666666667,
      "grad_norm": 1.9278602600097656,
      "learning_rate": 3.2976893231691424e-06,
      "logits/chosen": -1.750998854637146,
      "logits/rejected": -1.67935049533844,
      "logps/chosen": -232.449462890625,
      "logps/rejected": -352.84014892578125,
      "loss": 0.1109,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.335108757019043,
      "rewards/margins": 6.393424987792969,
      "rewards/rejected": -7.728533744812012,
      "step": 585
    },
    {
      "epoch": 1.2581333333333333,
      "grad_norm": 5.031718730926514,
      "learning_rate": 3.270351665601691e-06,
      "logits/chosen": -1.761778473854065,
      "logits/rejected": -1.628424048423767,
      "logps/chosen": -226.8080291748047,
      "logps/rejected": -330.947998046875,
      "loss": 0.1459,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.291100025177002,
      "rewards/margins": 6.421347141265869,
      "rewards/rejected": -7.712447166442871,
      "step": 590
    },
    {
      "epoch": 1.2688,
      "grad_norm": 2.0690762996673584,
      "learning_rate": 3.2429118446573417e-06,
      "logits/chosen": -1.797255277633667,
      "logits/rejected": -1.6581494808197021,
      "logps/chosen": -232.0042724609375,
      "logps/rejected": -345.2641296386719,
      "loss": 0.1162,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.5298513174057007,
      "rewards/margins": 6.056611061096191,
      "rewards/rejected": -7.586462497711182,
      "step": 595
    },
    {
      "epoch": 1.2794666666666665,
      "grad_norm": 2.8271262645721436,
      "learning_rate": 3.215373499381602e-06,
      "logits/chosen": -1.7107791900634766,
      "logits/rejected": -1.5847904682159424,
      "logps/chosen": -239.77529907226562,
      "logps/rejected": -327.42724609375,
      "loss": 0.0918,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.4036293029785156,
      "rewards/margins": 5.92539119720459,
      "rewards/rejected": -7.329021453857422,
      "step": 600
    },
    {
      "epoch": 1.2901333333333334,
      "grad_norm": 1.7590172290802002,
      "learning_rate": 3.1877402818861954e-06,
      "logits/chosen": -1.8445212841033936,
      "logits/rejected": -1.6904475688934326,
      "logps/chosen": -243.2404327392578,
      "logps/rejected": -349.3418273925781,
      "loss": 0.079,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8851490020751953,
      "rewards/margins": 6.981202602386475,
      "rewards/rejected": -8.866351127624512,
      "step": 605
    },
    {
      "epoch": 1.3008,
      "grad_norm": 1.688004970550537,
      "learning_rate": 3.1600158568647145e-06,
      "logits/chosen": -1.782779335975647,
      "logits/rejected": -1.7313483953475952,
      "logps/chosen": -218.34713745117188,
      "logps/rejected": -319.5534362792969,
      "loss": 0.0996,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.3257147073745728,
      "rewards/margins": 6.426403045654297,
      "rewards/rejected": -7.752118110656738,
      "step": 610
    },
    {
      "epoch": 1.3114666666666666,
      "grad_norm": 1.4718377590179443,
      "learning_rate": 3.1322039011066172e-06,
      "logits/chosen": -1.7013763189315796,
      "logits/rejected": -1.566279649734497,
      "logps/chosen": -247.23135375976562,
      "logps/rejected": -336.8480529785156,
      "loss": 0.0784,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.3434125185012817,
      "rewards/margins": 6.7102203369140625,
      "rewards/rejected": -8.053632736206055,
      "step": 615
    },
    {
      "epoch": 1.3221333333333334,
      "grad_norm": 3.314124822616577,
      "learning_rate": 3.1043081030096125e-06,
      "logits/chosen": -1.6063953638076782,
      "logits/rejected": -1.5149648189544678,
      "logps/chosen": -228.9906463623047,
      "logps/rejected": -318.33660888671875,
      "loss": 0.1127,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6745455265045166,
      "rewards/margins": 6.050885200500488,
      "rewards/rejected": -7.725429534912109,
      "step": 620
    },
    {
      "epoch": 1.3328,
      "grad_norm": 2.55005145072937,
      "learning_rate": 3.0763321620905072e-06,
      "logits/chosen": -1.6070985794067383,
      "logits/rejected": -1.5515340566635132,
      "logps/chosen": -219.2987823486328,
      "logps/rejected": -357.78619384765625,
      "loss": 0.1139,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6658519506454468,
      "rewards/margins": 6.835375785827637,
      "rewards/rejected": -8.501228332519531,
      "step": 625
    },
    {
      "epoch": 1.3434666666666666,
      "grad_norm": 1.571263074874878,
      "learning_rate": 3.048279788494583e-06,
      "logits/chosen": -1.7900960445404053,
      "logits/rejected": -1.6767871379852295,
      "logps/chosen": -217.8803253173828,
      "logps/rejected": -332.90838623046875,
      "loss": 0.097,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8096497058868408,
      "rewards/margins": 7.075921535491943,
      "rewards/rejected": -8.88557243347168,
      "step": 630
    },
    {
      "epoch": 1.3541333333333334,
      "grad_norm": 1.4549236297607422,
      "learning_rate": 3.0201547025035553e-06,
      "logits/chosen": -1.6474237442016602,
      "logits/rejected": -1.5805208683013916,
      "logps/chosen": -222.01071166992188,
      "logps/rejected": -363.7785339355469,
      "loss": 0.1013,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6705299615859985,
      "rewards/margins": 7.639910697937012,
      "rewards/rejected": -9.310441970825195,
      "step": 635
    },
    {
      "epoch": 1.3648,
      "grad_norm": 4.120782852172852,
      "learning_rate": 2.9919606340421963e-06,
      "logits/chosen": -1.8172228336334229,
      "logits/rejected": -1.6626869440078735,
      "logps/chosen": -214.86422729492188,
      "logps/rejected": -349.9516296386719,
      "loss": 0.1206,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9698278903961182,
      "rewards/margins": 7.807936191558838,
      "rewards/rejected": -9.777763366699219,
      "step": 640
    },
    {
      "epoch": 1.3754666666666666,
      "grad_norm": 0.9963375926017761,
      "learning_rate": 2.9637013221836723e-06,
      "logits/chosen": -1.753147840499878,
      "logits/rejected": -1.6141316890716553,
      "logps/chosen": -239.36892700195312,
      "logps/rejected": -333.98956298828125,
      "loss": 0.1611,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.883836030960083,
      "rewards/margins": 6.4801025390625,
      "rewards/rejected": -8.363938331604004,
      "step": 645
    },
    {
      "epoch": 1.3861333333333334,
      "grad_norm": 5.172098636627197,
      "learning_rate": 2.9353805146536762e-06,
      "logits/chosen": -1.6604881286621094,
      "logits/rejected": -1.6201698780059814,
      "logps/chosen": -220.64718627929688,
      "logps/rejected": -341.5318603515625,
      "loss": 0.148,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.7640655040740967,
      "rewards/margins": 6.463761806488037,
      "rewards/rejected": -8.227828025817871,
      "step": 650
    },
    {
      "epoch": 1.3968,
      "grad_norm": 2.365213632583618,
      "learning_rate": 2.907001967333401e-06,
      "logits/chosen": -1.8392598628997803,
      "logits/rejected": -1.7236486673355103,
      "logps/chosen": -228.3126678466797,
      "logps/rejected": -362.47869873046875,
      "loss": 0.122,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.6854579448699951,
      "rewards/margins": 7.8559770584106445,
      "rewards/rejected": -9.541436195373535,
      "step": 655
    },
    {
      "epoch": 1.4074666666666666,
      "grad_norm": 5.077343940734863,
      "learning_rate": 2.878569443761442e-06,
      "logits/chosen": -1.7001498937606812,
      "logits/rejected": -1.5788886547088623,
      "logps/chosen": -218.8361358642578,
      "logps/rejected": -336.6226501464844,
      "loss": 0.1438,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9872394800186157,
      "rewards/margins": 6.80609130859375,
      "rewards/rejected": -8.793330192565918,
      "step": 660
    },
    {
      "epoch": 1.4181333333333335,
      "grad_norm": 3.6902966499328613,
      "learning_rate": 2.85008671463468e-06,
      "logits/chosen": -1.6842687129974365,
      "logits/rejected": -1.6067196130752563,
      "logps/chosen": -217.57186889648438,
      "logps/rejected": -340.4504089355469,
      "loss": 0.1503,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.7623138427734375,
      "rewards/margins": 6.442144870758057,
      "rewards/rejected": -8.204458236694336,
      "step": 665
    },
    {
      "epoch": 1.4288,
      "grad_norm": 2.927492380142212,
      "learning_rate": 2.8215575573082127e-06,
      "logits/chosen": -1.7495641708374023,
      "logits/rejected": -1.673855185508728,
      "logps/chosen": -225.540771484375,
      "logps/rejected": -351.88665771484375,
      "loss": 0.0832,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.921282410621643,
      "rewards/margins": 6.98816442489624,
      "rewards/rejected": -8.909446716308594,
      "step": 670
    },
    {
      "epoch": 1.4394666666666667,
      "grad_norm": 1.9164410829544067,
      "learning_rate": 2.792985755294404e-06,
      "logits/chosen": -1.841391921043396,
      "logits/rejected": -1.6707547903060913,
      "logps/chosen": -209.87161254882812,
      "logps/rejected": -349.9462890625,
      "loss": 0.0892,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.7684261798858643,
      "rewards/margins": 7.643846035003662,
      "rewards/rejected": -9.412271499633789,
      "step": 675
    },
    {
      "epoch": 1.4501333333333333,
      "grad_norm": 2.0672764778137207,
      "learning_rate": 2.764375097761122e-06,
      "logits/chosen": -1.6338927745819092,
      "logits/rejected": -1.4459412097930908,
      "logps/chosen": -221.2202606201172,
      "logps/rejected": -330.55242919921875,
      "loss": 0.0878,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.3578730821609497,
      "rewards/margins": 7.062647819519043,
      "rewards/rejected": -8.420520782470703,
      "step": 680
    },
    {
      "epoch": 1.4607999999999999,
      "grad_norm": 1.4110078811645508,
      "learning_rate": 2.73572937902922e-06,
      "logits/chosen": -1.7500721216201782,
      "logits/rejected": -1.5899924039840698,
      "logps/chosen": -234.5608673095703,
      "logps/rejected": -351.0326232910156,
      "loss": 0.0846,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7048619985580444,
      "rewards/margins": 7.528687953948975,
      "rewards/rejected": -9.233550071716309,
      "step": 685
    },
    {
      "epoch": 1.4714666666666667,
      "grad_norm": NaN,
      "learning_rate": 2.7070523980693362e-06,
      "logits/chosen": -1.5613092184066772,
      "logits/rejected": -1.4905552864074707,
      "logps/chosen": -243.5371856689453,
      "logps/rejected": -338.37060546875,
      "loss": 0.0945,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.6675678491592407,
      "rewards/margins": 7.0091729164123535,
      "rewards/rejected": -8.676740646362305,
      "step": 690
    },
    {
      "epoch": 1.4821333333333333,
      "grad_norm": 3.837553024291992,
      "learning_rate": 2.678347957998081e-06,
      "logits/chosen": -1.741687536239624,
      "logits/rejected": -1.612644910812378,
      "logps/chosen": -207.15847778320312,
      "logps/rejected": -344.6006774902344,
      "loss": 0.0866,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7650225162506104,
      "rewards/margins": 7.5840888023376465,
      "rewards/rejected": -9.349112510681152,
      "step": 695
    },
    {
      "epoch": 1.4928,
      "grad_norm": 1.987453818321228,
      "learning_rate": 2.649619865573666e-06,
      "logits/chosen": -1.6311708688735962,
      "logits/rejected": -1.6032650470733643,
      "logps/chosen": -224.74313354492188,
      "logps/rejected": -356.2095642089844,
      "loss": 0.0758,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1680212020874023,
      "rewards/margins": 7.786881446838379,
      "rewards/rejected": -9.954902648925781,
      "step": 700
    },
    {
      "epoch": 1.5034666666666667,
      "grad_norm": 2.1548123359680176,
      "learning_rate": 2.620871930691058e-06,
      "logits/chosen": -1.7811607122421265,
      "logits/rejected": -1.7126588821411133,
      "logps/chosen": -275.605712890625,
      "logps/rejected": -376.85308837890625,
      "loss": 0.0698,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.9767110347747803,
      "rewards/margins": 7.755047798156738,
      "rewards/rejected": -9.731759071350098,
      "step": 705
    },
    {
      "epoch": 1.5141333333333333,
      "grad_norm": 3.925725221633911,
      "learning_rate": 2.5921079658767152e-06,
      "logits/chosen": -1.7364921569824219,
      "logits/rejected": -1.5872316360473633,
      "logps/chosen": -219.77951049804688,
      "logps/rejected": -333.9054870605469,
      "loss": 0.095,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.9969074726104736,
      "rewards/margins": 7.254309177398682,
      "rewards/rejected": -9.251215934753418,
      "step": 710
    },
    {
      "epoch": 1.5248,
      "grad_norm": 3.2879045009613037,
      "learning_rate": 2.56333178578297e-06,
      "logits/chosen": -1.744199514389038,
      "logits/rejected": -1.5770639181137085,
      "logps/chosen": -217.22317504882812,
      "logps/rejected": -348.82867431640625,
      "loss": 0.1105,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.2446813583374023,
      "rewards/margins": 7.685030460357666,
      "rewards/rejected": -9.92971134185791,
      "step": 715
    },
    {
      "epoch": 1.5354666666666668,
      "grad_norm": 4.384338855743408,
      "learning_rate": 2.534547206682136e-06,
      "logits/chosen": -1.691664457321167,
      "logits/rejected": -1.6016451120376587,
      "logps/chosen": -232.01016235351562,
      "logps/rejected": -377.1441345214844,
      "loss": 0.0828,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.3139617443084717,
      "rewards/margins": 8.008916854858398,
      "rewards/rejected": -10.32287883758545,
      "step": 720
    },
    {
      "epoch": 1.5461333333333334,
      "grad_norm": 1.2283564805984497,
      "learning_rate": 2.5057580459603947e-06,
      "logits/chosen": -1.7561960220336914,
      "logits/rejected": -1.623084306716919,
      "logps/chosen": -205.0697784423828,
      "logps/rejected": -363.44268798828125,
      "loss": 0.1036,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.9934345483779907,
      "rewards/margins": 8.515266418457031,
      "rewards/rejected": -10.508700370788574,
      "step": 725
    },
    {
      "epoch": 1.5568,
      "grad_norm": 2.0477917194366455,
      "learning_rate": 2.4769681216115384e-06,
      "logits/chosen": -1.770697832107544,
      "logits/rejected": -1.6845451593399048,
      "logps/chosen": -223.17129516601562,
      "logps/rejected": -366.52569580078125,
      "loss": 0.0925,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7600250244140625,
      "rewards/margins": 8.27499771118164,
      "rewards/rejected": -10.03502368927002,
      "step": 730
    },
    {
      "epoch": 1.5674666666666668,
      "grad_norm": 3.279738187789917,
      "learning_rate": 2.448181251730632e-06,
      "logits/chosen": -1.71505606174469,
      "logits/rejected": -1.6390784978866577,
      "logps/chosen": -233.17410278320312,
      "logps/rejected": -355.71905517578125,
      "loss": 0.1536,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -2.3069491386413574,
      "rewards/margins": 7.33617639541626,
      "rewards/rejected": -9.643125534057617,
      "step": 735
    },
    {
      "epoch": 1.5781333333333334,
      "grad_norm": 3.533376693725586,
      "learning_rate": 2.419401254007658e-06,
      "logits/chosen": -1.6508073806762695,
      "logits/rejected": -1.5678499937057495,
      "logps/chosen": -246.8570556640625,
      "logps/rejected": -379.45648193359375,
      "loss": 0.0884,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.69010329246521,
      "rewards/margins": 7.795050621032715,
      "rewards/rejected": -10.48515510559082,
      "step": 740
    },
    {
      "epoch": 1.5888,
      "grad_norm": 1.8679414987564087,
      "learning_rate": 2.390631945221221e-06,
      "logits/chosen": -1.6903434991836548,
      "logits/rejected": -1.565431833267212,
      "logps/chosen": -242.61788940429688,
      "logps/rejected": -357.1767578125,
      "loss": 0.0628,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.191249370574951,
      "rewards/margins": 8.514483451843262,
      "rewards/rejected": -10.705734252929688,
      "step": 745
    },
    {
      "epoch": 1.5994666666666668,
      "grad_norm": 2.0882701873779297,
      "learning_rate": 2.361877140732366e-06,
      "logits/chosen": -1.691928505897522,
      "logits/rejected": -1.5991172790527344,
      "logps/chosen": -222.71511840820312,
      "logps/rejected": -371.5699462890625,
      "loss": 0.0904,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.2086446285247803,
      "rewards/margins": 8.20213508605957,
      "rewards/rejected": -10.410778999328613,
      "step": 750
    },
    {
      "epoch": 1.6101333333333332,
      "grad_norm": 3.39593505859375,
      "learning_rate": 2.333140653978592e-06,
      "logits/chosen": -1.7964928150177002,
      "logits/rejected": -1.6038188934326172,
      "logps/chosen": -221.6672821044922,
      "logps/rejected": -335.6654357910156,
      "loss": 0.0802,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.082045316696167,
      "rewards/margins": 8.015445709228516,
      "rewards/rejected": -10.097490310668945,
      "step": 755
    },
    {
      "epoch": 1.6208,
      "grad_norm": 5.544408798217773,
      "learning_rate": 2.3044262959681156e-06,
      "logits/chosen": -1.7598270177841187,
      "logits/rejected": -1.5819107294082642,
      "logps/chosen": -222.6882781982422,
      "logps/rejected": -347.28985595703125,
      "loss": 0.1282,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -1.9872537851333618,
      "rewards/margins": 7.4105939865112305,
      "rewards/rejected": -9.397848129272461,
      "step": 760
    },
    {
      "epoch": 1.6314666666666666,
      "grad_norm": 1.4111205339431763,
      "learning_rate": 2.2757378747744575e-06,
      "logits/chosen": -1.763664960861206,
      "logits/rejected": -1.5994547605514526,
      "logps/chosen": -218.54043579101562,
      "logps/rejected": -361.8656921386719,
      "loss": 0.0953,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -1.9402878284454346,
      "rewards/margins": 8.229757308959961,
      "rewards/rejected": -10.170045852661133,
      "step": 765
    },
    {
      "epoch": 1.6421333333333332,
      "grad_norm": 4.1563215255737305,
      "learning_rate": 2.24707919503142e-06,
      "logits/chosen": -1.773432970046997,
      "logits/rejected": -1.6846001148223877,
      "logps/chosen": -217.88540649414062,
      "logps/rejected": -382.12823486328125,
      "loss": 0.0899,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.1007978916168213,
      "rewards/margins": 8.273159980773926,
      "rewards/rejected": -10.373956680297852,
      "step": 770
    },
    {
      "epoch": 1.6528,
      "grad_norm": 2.1946141719818115,
      "learning_rate": 2.2184540574285213e-06,
      "logits/chosen": -1.786668062210083,
      "logits/rejected": -1.6237224340438843,
      "logps/chosen": -258.10015869140625,
      "logps/rejected": -369.31195068359375,
      "loss": 0.1073,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0337066650390625,
      "rewards/margins": 7.8874945640563965,
      "rewards/rejected": -9.921201705932617,
      "step": 775
    },
    {
      "epoch": 1.6634666666666666,
      "grad_norm": 3.9432976245880127,
      "learning_rate": 2.1898662582069486e-06,
      "logits/chosen": -1.7747735977172852,
      "logits/rejected": -1.6449607610702515,
      "logps/chosen": -211.96719360351562,
      "logps/rejected": -374.8441467285156,
      "loss": 0.0624,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5460779666900635,
      "rewards/margins": 8.801756858825684,
      "rewards/rejected": -10.347834587097168,
      "step": 780
    },
    {
      "epoch": 1.6741333333333333,
      "grad_norm": 2.114940643310547,
      "learning_rate": 2.1613195886561044e-06,
      "logits/chosen": -1.797666311264038,
      "logits/rejected": -1.6107561588287354,
      "logps/chosen": -219.36013793945312,
      "logps/rejected": -354.28033447265625,
      "loss": 0.0811,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.8233768939971924,
      "rewards/margins": 8.105354309082031,
      "rewards/rejected": -9.928730964660645,
      "step": 785
    },
    {
      "epoch": 1.6848,
      "grad_norm": 1.5351241827011108,
      "learning_rate": 2.132817834610811e-06,
      "logits/chosen": -1.751610517501831,
      "logits/rejected": -1.5730657577514648,
      "logps/chosen": -222.4546356201172,
      "logps/rejected": -355.1824035644531,
      "loss": 0.0647,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -1.9879817962646484,
      "rewards/margins": 8.064549446105957,
      "rewards/rejected": -10.052530288696289,
      "step": 790
    },
    {
      "epoch": 1.6954666666666667,
      "grad_norm": 3.84106183052063,
      "learning_rate": 2.1043647759492338e-06,
      "logits/chosen": -1.5872787237167358,
      "logits/rejected": -1.5767438411712646,
      "logps/chosen": -239.85684204101562,
      "logps/rejected": -348.74920654296875,
      "loss": 0.0847,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.5143554210662842,
      "rewards/margins": 7.725428581237793,
      "rewards/rejected": -9.239785194396973,
      "step": 795
    },
    {
      "epoch": 1.7061333333333333,
      "grad_norm": 3.106780767440796,
      "learning_rate": 2.075964186091599e-06,
      "logits/chosen": -1.7316697835922241,
      "logits/rejected": -1.650206208229065,
      "logps/chosen": -226.59616088867188,
      "logps/rejected": -354.37823486328125,
      "loss": 0.093,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.121168613433838,
      "rewards/margins": 7.989222526550293,
      "rewards/rejected": -10.110391616821289,
      "step": 800
    },
    {
      "epoch": 1.7168,
      "grad_norm": 1.7571396827697754,
      "learning_rate": 2.047619831499765e-06,
      "logits/chosen": -1.7232033014297485,
      "logits/rejected": -1.568366289138794,
      "logps/chosen": -234.1296844482422,
      "logps/rejected": -346.7981872558594,
      "loss": 0.1052,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1577060222625732,
      "rewards/margins": 7.079756736755371,
      "rewards/rejected": -9.237462997436523,
      "step": 805
    },
    {
      "epoch": 1.7274666666666667,
      "grad_norm": 2.546520948410034,
      "learning_rate": 2.0193354711777155e-06,
      "logits/chosen": -1.7943166494369507,
      "logits/rejected": -1.6009585857391357,
      "logps/chosen": -238.33779907226562,
      "logps/rejected": -359.30426025390625,
      "loss": 0.0945,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.741201639175415,
      "rewards/margins": 8.233468055725098,
      "rewards/rejected": -9.974669456481934,
      "step": 810
    },
    {
      "epoch": 1.7381333333333333,
      "grad_norm": 4.782870769500732,
      "learning_rate": 1.991114856173048e-06,
      "logits/chosen": -1.7833083868026733,
      "logits/rejected": -1.6827278137207031,
      "logps/chosen": -235.49661254882812,
      "logps/rejected": -378.20562744140625,
      "loss": 0.0973,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.7793164253234863,
      "rewards/margins": 8.449636459350586,
      "rewards/rejected": -11.228952407836914,
      "step": 815
    },
    {
      "epoch": 1.7488000000000001,
      "grad_norm": 1.997058391571045,
      "learning_rate": 1.9629617290795115e-06,
      "logits/chosen": -1.6820003986358643,
      "logits/rejected": -1.6001437902450562,
      "logps/chosen": -212.8640594482422,
      "logps/rejected": -348.04241943359375,
      "loss": 0.0829,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.0805208683013916,
      "rewards/margins": 7.856542110443115,
      "rewards/rejected": -9.937063217163086,
      "step": 820
    },
    {
      "epoch": 1.7594666666666665,
      "grad_norm": 1.1324129104614258,
      "learning_rate": 1.934879823540663e-06,
      "logits/chosen": -1.7404649257659912,
      "logits/rejected": -1.5713565349578857,
      "logps/chosen": -217.12887573242188,
      "logps/rejected": -357.5457458496094,
      "loss": 0.0789,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.7752182483673096,
      "rewards/margins": 8.235331535339355,
      "rewards/rejected": -10.01054859161377,
      "step": 825
    },
    {
      "epoch": 1.7701333333333333,
      "grad_norm": 0.8319897651672363,
      "learning_rate": 1.9068728637547195e-06,
      "logits/chosen": -1.713891625404358,
      "logits/rejected": -1.6125208139419556,
      "logps/chosen": -236.43612670898438,
      "logps/rejected": -374.36236572265625,
      "loss": 0.0656,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.294593572616577,
      "rewards/margins": 8.278243064880371,
      "rewards/rejected": -10.572836875915527,
      "step": 830
    },
    {
      "epoch": 1.7808000000000002,
      "grad_norm": 1.1885546445846558,
      "learning_rate": 1.878944563980658e-06,
      "logits/chosen": -1.659051537513733,
      "logits/rejected": -1.5556640625,
      "logps/chosen": -217.66622924804688,
      "logps/rejected": -363.8323669433594,
      "loss": 0.0698,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.268317461013794,
      "rewards/margins": 7.98989200592041,
      "rewards/rejected": -10.258210182189941,
      "step": 835
    },
    {
      "epoch": 1.7914666666666665,
      "grad_norm": 3.575385332107544,
      "learning_rate": 1.85109862804563e-06,
      "logits/chosen": -1.717116117477417,
      "logits/rejected": -1.5746225118637085,
      "logps/chosen": -245.6321258544922,
      "logps/rejected": -334.73565673828125,
      "loss": 0.0874,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.0019404888153076,
      "rewards/margins": 7.676904201507568,
      "rewards/rejected": -9.678845405578613,
      "step": 840
    },
    {
      "epoch": 1.8021333333333334,
      "grad_norm": 1.076490044593811,
      "learning_rate": 1.823338748853766e-06,
      "logits/chosen": -1.7594982385635376,
      "logits/rejected": -1.6361639499664307,
      "logps/chosen": -230.13330078125,
      "logps/rejected": -371.546875,
      "loss": 0.0682,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2079403400421143,
      "rewards/margins": 8.9056396484375,
      "rewards/rejected": -11.113580703735352,
      "step": 845
    },
    {
      "epoch": 1.8128,
      "grad_norm": 1.023444414138794,
      "learning_rate": 1.7956686078964257e-06,
      "logits/chosen": -1.785599946975708,
      "logits/rejected": -1.6359217166900635,
      "logps/chosen": -234.2008819580078,
      "logps/rejected": -333.7929992675781,
      "loss": 0.1263,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.184248447418213,
      "rewards/margins": 7.763758659362793,
      "rewards/rejected": -9.948007583618164,
      "step": 850
    },
    {
      "epoch": 1.8234666666666666,
      "grad_norm": 5.103050231933594,
      "learning_rate": 1.7680918747639608e-06,
      "logits/chosen": -1.7238714694976807,
      "logits/rejected": -1.6119329929351807,
      "logps/chosen": -235.1346893310547,
      "logps/rejected": -385.647705078125,
      "loss": 0.0734,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.530133008956909,
      "rewards/margins": 8.610017776489258,
      "rewards/rejected": -11.14015007019043,
      "step": 855
    },
    {
      "epoch": 1.8341333333333334,
      "grad_norm": 2.734606981277466,
      "learning_rate": 1.740612206659057e-06,
      "logits/chosen": -1.798112154006958,
      "logits/rejected": -1.5648561716079712,
      "logps/chosen": -229.7447052001953,
      "logps/rejected": -367.8794860839844,
      "loss": 0.0698,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2181127071380615,
      "rewards/margins": 8.9027681350708,
      "rewards/rejected": -11.120881080627441,
      "step": 860
    },
    {
      "epoch": 1.8448,
      "grad_norm": 1.9527674913406372,
      "learning_rate": 1.7132332479117204e-06,
      "logits/chosen": -1.780371904373169,
      "logits/rejected": -1.5965863466262817,
      "logps/chosen": -245.5487518310547,
      "logps/rejected": -359.4418029785156,
      "loss": 0.1013,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1059577465057373,
      "rewards/margins": 8.449846267700195,
      "rewards/rejected": -10.555803298950195,
      "step": 865
    },
    {
      "epoch": 1.8554666666666666,
      "grad_norm": 6.878162384033203,
      "learning_rate": 1.6859586294959666e-06,
      "logits/chosen": -1.7246229648590088,
      "logits/rejected": -1.6141942739486694,
      "logps/chosen": -234.10302734375,
      "logps/rejected": -347.4426574707031,
      "loss": 0.0942,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0066280364990234,
      "rewards/margins": 8.30981731414795,
      "rewards/rejected": -10.316446304321289,
      "step": 870
    },
    {
      "epoch": 1.8661333333333334,
      "grad_norm": 2.037459135055542,
      "learning_rate": 1.6587919685482884e-06,
      "logits/chosen": -1.8677127361297607,
      "logits/rejected": -1.7134828567504883,
      "logps/chosen": -212.8134002685547,
      "logps/rejected": -393.59124755859375,
      "loss": 0.0808,
      "rewards/accuracies": 0.956250011920929,
      "rewards/chosen": -2.3060128688812256,
      "rewards/margins": 8.69517993927002,
      "rewards/rejected": -11.001192092895508,
      "step": 875
    },
    {
      "epoch": 1.8768,
      "grad_norm": 1.5264317989349365,
      "learning_rate": 1.6317368678879497e-06,
      "logits/chosen": -1.6491420269012451,
      "logits/rejected": -1.5571231842041016,
      "logps/chosen": -231.6708526611328,
      "logps/rejected": -384.9656677246094,
      "loss": 0.0894,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.254824161529541,
      "rewards/margins": 9.314870834350586,
      "rewards/rejected": -11.569695472717285,
      "step": 880
    },
    {
      "epoch": 1.8874666666666666,
      "grad_norm": 2.8329501152038574,
      "learning_rate": 1.6047969155391856e-06,
      "logits/chosen": -1.6241254806518555,
      "logits/rejected": -1.6059930324554443,
      "logps/chosen": -242.1722412109375,
      "logps/rejected": -375.7530517578125,
      "loss": 0.0708,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.215430498123169,
      "rewards/margins": 8.22697639465332,
      "rewards/rejected": -10.442407608032227,
      "step": 885
    },
    {
      "epoch": 1.8981333333333335,
      "grad_norm": 1.2483875751495361,
      "learning_rate": 1.5779756842553617e-06,
      "logits/chosen": -1.695774793624878,
      "logits/rejected": -1.667642593383789,
      "logps/chosen": -236.163330078125,
      "logps/rejected": -370.6156311035156,
      "loss": 0.0599,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2344157695770264,
      "rewards/margins": 8.541747093200684,
      "rewards/rejected": -10.776163101196289,
      "step": 890
    },
    {
      "epoch": 1.9088,
      "grad_norm": 3.0992672443389893,
      "learning_rate": 1.551276731045157e-06,
      "logits/chosen": -1.7390620708465576,
      "logits/rejected": -1.6380354166030884,
      "logps/chosen": -248.9411163330078,
      "logps/rejected": -380.0399475097656,
      "loss": 0.0641,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2495522499084473,
      "rewards/margins": 8.640344619750977,
      "rewards/rejected": -10.889896392822266,
      "step": 895
    },
    {
      "epoch": 1.9194666666666667,
      "grad_norm": 1.3069320917129517,
      "learning_rate": 1.5247035967008369e-06,
      "logits/chosen": -1.6581281423568726,
      "logits/rejected": -1.498859167098999,
      "logps/chosen": -232.68618774414062,
      "logps/rejected": -349.40374755859375,
      "loss": 0.0631,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9037091732025146,
      "rewards/margins": 8.36854362487793,
      "rewards/rejected": -10.272253036499023,
      "step": 900
    },
    {
      "epoch": 1.9301333333333335,
      "grad_norm": 3.359713077545166,
      "learning_rate": 1.498259805328678e-06,
      "logits/chosen": -1.6838054656982422,
      "logits/rejected": -1.529866337776184,
      "logps/chosen": -230.3268585205078,
      "logps/rejected": -340.8519287109375,
      "loss": 0.0802,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.370974063873291,
      "rewards/margins": 8.133573532104492,
      "rewards/rejected": -10.504546165466309,
      "step": 905
    },
    {
      "epoch": 1.9407999999999999,
      "grad_norm": 1.0852340459823608,
      "learning_rate": 1.471948863881601e-06,
      "logits/chosen": -1.7539818286895752,
      "logits/rejected": -1.6175880432128906,
      "logps/chosen": -244.2122039794922,
      "logps/rejected": -387.53924560546875,
      "loss": 0.0534,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.2629380226135254,
      "rewards/margins": 8.641889572143555,
      "rewards/rejected": -10.904828071594238,
      "step": 910
    },
    {
      "epoch": 1.9514666666666667,
      "grad_norm": 3.436288356781006,
      "learning_rate": 1.4457742616940864e-06,
      "logits/chosen": -1.6151090860366821,
      "logits/rejected": -1.5289099216461182,
      "logps/chosen": -245.4002227783203,
      "logps/rejected": -371.73565673828125,
      "loss": 0.0941,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.361675500869751,
      "rewards/margins": 7.663311958312988,
      "rewards/rejected": -10.02498722076416,
      "step": 915
    },
    {
      "epoch": 1.9621333333333333,
      "grad_norm": 3.603616237640381,
      "learning_rate": 1.4197394700194165e-06,
      "logits/chosen": -1.7797037363052368,
      "logits/rejected": -1.6602210998535156,
      "logps/chosen": -229.91262817382812,
      "logps/rejected": -347.8226013183594,
      "loss": 0.0738,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.6596853733062744,
      "rewards/margins": 7.811383247375488,
      "rewards/rejected": -10.471068382263184,
      "step": 920
    },
    {
      "epoch": 1.9727999999999999,
      "grad_norm": 5.25838565826416,
      "learning_rate": 1.3938479415693244e-06,
      "logits/chosen": -1.7488892078399658,
      "logits/rejected": -1.6513652801513672,
      "logps/chosen": -237.98727416992188,
      "logps/rejected": -373.8163757324219,
      "loss": 0.0797,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.413729667663574,
      "rewards/margins": 9.135477066040039,
      "rewards/rejected": -11.54920768737793,
      "step": 925
    },
    {
      "epoch": 1.9834666666666667,
      "grad_norm": 4.8256425857543945,
      "learning_rate": 1.368103110056096e-06,
      "logits/chosen": -1.7451817989349365,
      "logits/rejected": -1.6328808069229126,
      "logps/chosen": -253.07363891601562,
      "logps/rejected": -370.066650390625,
      "loss": 0.1011,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.8300654888153076,
      "rewards/margins": 8.426507949829102,
      "rewards/rejected": -11.256572723388672,
      "step": 930
    },
    {
      "epoch": 1.9941333333333333,
      "grad_norm": 0.881809413433075,
      "learning_rate": 1.3425083897371983e-06,
      "logits/chosen": -1.5809837579727173,
      "logits/rejected": -1.5179001092910767,
      "logps/chosen": -246.60287475585938,
      "logps/rejected": -365.4212341308594,
      "loss": 0.0904,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0594959259033203,
      "rewards/margins": 8.06749153137207,
      "rewards/rejected": -10.126986503601074,
      "step": 935
    },
    {
      "epoch": 2.0042666666666666,
      "grad_norm": 3.4227020740509033,
      "learning_rate": 1.317067174962478e-06,
      "logits/chosen": -1.6698054075241089,
      "logits/rejected": -1.4948031902313232,
      "logps/chosen": -241.87635803222656,
      "logps/rejected": -372.1206970214844,
      "loss": 0.0836,
      "rewards/accuracies": 0.9868420958518982,
      "rewards/chosen": -2.477794647216797,
      "rewards/margins": 8.646845817565918,
      "rewards/rejected": -11.124640464782715,
      "step": 940
    },
    {
      "epoch": 2.0149333333333335,
      "grad_norm": 2.4874966144561768,
      "learning_rate": 1.291782839724009e-06,
      "logits/chosen": -1.6582422256469727,
      "logits/rejected": -1.6185009479522705,
      "logps/chosen": -228.345947265625,
      "logps/rejected": -355.91973876953125,
      "loss": 0.073,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.1200289726257324,
      "rewards/margins": 8.30976390838623,
      "rewards/rejected": -10.429792404174805,
      "step": 945
    },
    {
      "epoch": 2.0256,
      "grad_norm": 1.4161961078643799,
      "learning_rate": 1.2666587372086352e-06,
      "logits/chosen": -1.6435474157333374,
      "logits/rejected": -1.5627238750457764,
      "logps/chosen": -209.0333709716797,
      "logps/rejected": -350.1138916015625,
      "loss": 0.0693,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.8323615789413452,
      "rewards/margins": 8.273909568786621,
      "rewards/rejected": -10.106269836425781,
      "step": 950
    },
    {
      "epoch": 2.0362666666666667,
      "grad_norm": 3.069528102874756,
      "learning_rate": 1.241698199353275e-06,
      "logits/chosen": -1.7319631576538086,
      "logits/rejected": -1.5979344844818115,
      "logps/chosen": -216.0243377685547,
      "logps/rejected": -368.0318298339844,
      "loss": 0.0599,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.209562301635742,
      "rewards/margins": 8.545756340026855,
      "rewards/rejected": -10.755319595336914,
      "step": 955
    },
    {
      "epoch": 2.0469333333333335,
      "grad_norm": 5.300919532775879,
      "learning_rate": 1.2169045364030404e-06,
      "logits/chosen": -1.7645591497421265,
      "logits/rejected": -1.62359619140625,
      "logps/chosen": -227.6970672607422,
      "logps/rejected": -369.8218078613281,
      "loss": 0.0689,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6951305866241455,
      "rewards/margins": 8.503260612487793,
      "rewards/rejected": -11.19839096069336,
      "step": 960
    },
    {
      "epoch": 2.0576,
      "grad_norm": 2.6272425651550293,
      "learning_rate": 1.1922810364722402e-06,
      "logits/chosen": -1.7294490337371826,
      "logits/rejected": -1.5723257064819336,
      "logps/chosen": -243.034423828125,
      "logps/rejected": -356.21685791015625,
      "loss": 0.0824,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.0579140186309814,
      "rewards/margins": 8.599225044250488,
      "rewards/rejected": -10.65713882446289,
      "step": 965
    },
    {
      "epoch": 2.0682666666666667,
      "grad_norm": 3.7131705284118652,
      "learning_rate": 1.1678309651083055e-06,
      "logits/chosen": -1.7538963556289673,
      "logits/rejected": -1.6433712244033813,
      "logps/chosen": -242.6889190673828,
      "logps/rejected": -358.15277099609375,
      "loss": 0.0524,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.461589813232422,
      "rewards/margins": 8.619664192199707,
      "rewards/rejected": -11.081254959106445,
      "step": 970
    },
    {
      "epoch": 2.0789333333333335,
      "grad_norm": 6.3900909423828125,
      "learning_rate": 1.1435575648587224e-06,
      "logits/chosen": -1.6953847408294678,
      "logits/rejected": -1.621117353439331,
      "logps/chosen": -233.9971923828125,
      "logps/rejected": -375.391845703125,
      "loss": 0.0777,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4303104877471924,
      "rewards/margins": 9.11983585357666,
      "rewards/rejected": -11.550145149230957,
      "step": 975
    },
    {
      "epoch": 2.0896,
      "grad_norm": 0.9288052916526794,
      "learning_rate": 1.1194640548410039e-06,
      "logits/chosen": -1.6072397232055664,
      "logits/rejected": -1.5468966960906982,
      "logps/chosen": -229.78286743164062,
      "logps/rejected": -353.2090759277344,
      "loss": 0.0697,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.165677547454834,
      "rewards/margins": 8.486506462097168,
      "rewards/rejected": -10.65218448638916,
      "step": 980
    },
    {
      "epoch": 2.1002666666666667,
      "grad_norm": 1.0610967874526978,
      "learning_rate": 1.095553630315774e-06,
      "logits/chosen": -1.7196861505508423,
      "logits/rejected": -1.606482744216919,
      "logps/chosen": -221.6151580810547,
      "logps/rejected": -366.8529052734375,
      "loss": 0.0894,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -1.9320646524429321,
      "rewards/margins": 8.738572120666504,
      "rewards/rejected": -10.670637130737305,
      "step": 985
    },
    {
      "epoch": 2.1109333333333336,
      "grad_norm": 1.3458142280578613,
      "learning_rate": 1.0718294622630188e-06,
      "logits/chosen": -1.7000640630722046,
      "logits/rejected": -1.6278724670410156,
      "logps/chosen": -229.53878784179688,
      "logps/rejected": -381.9530029296875,
      "loss": 0.0627,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.1144888401031494,
      "rewards/margins": 8.574979782104492,
      "rewards/rejected": -10.689468383789062,
      "step": 990
    },
    {
      "epoch": 2.1216,
      "grad_norm": 2.989312171936035,
      "learning_rate": 1.0482946969615495e-06,
      "logits/chosen": -1.7484781742095947,
      "logits/rejected": -1.588270664215088,
      "logps/chosen": -246.27987670898438,
      "logps/rejected": -367.8683166503906,
      "loss": 0.0533,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7404978275299072,
      "rewards/margins": 8.548757553100586,
      "rewards/rejected": -11.289255142211914,
      "step": 995
    },
    {
      "epoch": 2.1322666666666668,
      "grad_norm": 1.4040266275405884,
      "learning_rate": 1.0249524555717461e-06,
      "logits/chosen": -1.7742398977279663,
      "logits/rejected": -1.6097848415374756,
      "logps/chosen": -223.71377563476562,
      "logps/rejected": -370.94305419921875,
      "loss": 0.0799,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.451437473297119,
      "rewards/margins": 8.23726749420166,
      "rewards/rejected": -10.688704490661621,
      "step": 1000
    },
    {
      "epoch": 2.142933333333333,
      "grad_norm": 1.852662444114685,
      "learning_rate": 1.0018058337216327e-06,
      "logits/chosen": -1.6842467784881592,
      "logits/rejected": -1.6574596166610718,
      "logps/chosen": -249.32254028320312,
      "logps/rejected": -369.68927001953125,
      "loss": 0.0656,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.577439785003662,
      "rewards/margins": 8.032537460327148,
      "rewards/rejected": -10.609979629516602,
      "step": 1005
    },
    {
      "epoch": 2.1536,
      "grad_norm": 3.422244071960449,
      "learning_rate": 9.788579010963387e-07,
      "logits/chosen": -1.7132316827774048,
      "logits/rejected": -1.5610625743865967,
      "logps/chosen": -246.59152221679688,
      "logps/rejected": -363.26898193359375,
      "loss": 0.0596,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5583107471466064,
      "rewards/margins": 8.616121292114258,
      "rewards/rejected": -11.174431800842285,
      "step": 1010
    },
    {
      "epoch": 2.164266666666667,
      "grad_norm": 3.149967908859253,
      "learning_rate": 9.56111701030998e-07,
      "logits/chosen": -1.8254941701889038,
      "logits/rejected": -1.6014741659164429,
      "logps/chosen": -257.9132080078125,
      "logps/rejected": -370.2958984375,
      "loss": 0.0748,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.586059093475342,
      "rewards/margins": 8.62619400024414,
      "rewards/rejected": -11.212254524230957,
      "step": 1015
    },
    {
      "epoch": 2.174933333333333,
      "grad_norm": 0.670924723148346,
      "learning_rate": 9.335702501071459e-07,
      "logits/chosen": -1.6369680166244507,
      "logits/rejected": -1.4811519384384155,
      "logps/chosen": -254.5886993408203,
      "logps/rejected": -366.7772216796875,
      "loss": 0.0637,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6869049072265625,
      "rewards/margins": 8.372596740722656,
      "rewards/rejected": -11.059501647949219,
      "step": 1020
    },
    {
      "epoch": 2.1856,
      "grad_norm": 0.4992627203464508,
      "learning_rate": 9.112365377526636e-07,
      "logits/chosen": -1.7114737033843994,
      "logits/rejected": -1.6579517126083374,
      "logps/chosen": -232.11563110351562,
      "logps/rejected": -392.6571350097656,
      "loss": 0.0379,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.052781581878662,
      "rewards/margins": 9.346757888793945,
      "rewards/rejected": -12.39953899383545,
      "step": 1025
    },
    {
      "epoch": 2.196266666666667,
      "grad_norm": 0.7529945373535156,
      "learning_rate": 8.891135258453195e-07,
      "logits/chosen": -1.6109974384307861,
      "logits/rejected": -1.518336534500122,
      "logps/chosen": -221.02468872070312,
      "logps/rejected": -362.6771240234375,
      "loss": 0.0525,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.2659616470336914,
      "rewards/margins": 9.202993392944336,
      "rewards/rejected": -11.468955993652344,
      "step": 1030
    },
    {
      "epoch": 2.206933333333333,
      "grad_norm": 2.34568190574646,
      "learning_rate": 8.672041483199709e-07,
      "logits/chosen": -1.7083330154418945,
      "logits/rejected": -1.5656239986419678,
      "logps/chosen": -244.4073486328125,
      "logps/rejected": -343.1373291015625,
      "loss": 0.0595,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.3901219367980957,
      "rewards/margins": 7.6386895179748535,
      "rewards/rejected": -10.028810501098633,
      "step": 1035
    },
    {
      "epoch": 2.2176,
      "grad_norm": 0.49317407608032227,
      "learning_rate": 8.455113107794652e-07,
      "logits/chosen": -1.6537469625473022,
      "logits/rejected": -1.6144192218780518,
      "logps/chosen": -217.38223266601562,
      "logps/rejected": -389.8377685546875,
      "loss": 0.0596,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.900716781616211,
      "rewards/margins": 9.58027458190918,
      "rewards/rejected": -12.480993270874023,
      "step": 1040
    },
    {
      "epoch": 2.228266666666667,
      "grad_norm": 2.7996599674224854,
      "learning_rate": 8.240378901093035e-07,
      "logits/chosen": -1.7323932647705078,
      "logits/rejected": -1.5816843509674072,
      "logps/chosen": -245.81143188476562,
      "logps/rejected": -368.0496826171875,
      "loss": 0.0609,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.495245933532715,
      "rewards/margins": 8.47819995880127,
      "rewards/rejected": -10.973443984985352,
      "step": 1045
    },
    {
      "epoch": 2.238933333333333,
      "grad_norm": 0.7402082681655884,
      "learning_rate": 8.027867340961113e-07,
      "logits/chosen": -1.5743385553359985,
      "logits/rejected": -1.4896413087844849,
      "logps/chosen": -235.98117065429688,
      "logps/rejected": -354.88763427734375,
      "loss": 0.0608,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.446408987045288,
      "rewards/margins": 8.018033981323242,
      "rewards/rejected": -10.464442253112793,
      "step": 1050
    },
    {
      "epoch": 2.2496,
      "grad_norm": 1.0578789710998535,
      "learning_rate": 7.817606610499639e-07,
      "logits/chosen": -1.7072864770889282,
      "logits/rejected": -1.6435352563858032,
      "logps/chosen": -240.1691436767578,
      "logps/rejected": -382.712158203125,
      "loss": 0.0511,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5651392936706543,
      "rewards/margins": 8.454736709594727,
      "rewards/rejected": -11.019875526428223,
      "step": 1055
    },
    {
      "epoch": 2.2602666666666664,
      "grad_norm": 0.5394600629806519,
      "learning_rate": 7.609624594306278e-07,
      "logits/chosen": -1.7428966760635376,
      "logits/rejected": -1.643012285232544,
      "logps/chosen": -247.66360473632812,
      "logps/rejected": -390.7719421386719,
      "loss": 0.0475,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6858017444610596,
      "rewards/margins": 8.323606491088867,
      "rewards/rejected": -11.009407043457031,
      "step": 1060
    },
    {
      "epoch": 2.2709333333333332,
      "grad_norm": 7.077690124511719,
      "learning_rate": 7.403948874777556e-07,
      "logits/chosen": -1.6925182342529297,
      "logits/rejected": -1.5964515209197998,
      "logps/chosen": -229.00881958007812,
      "logps/rejected": -373.58740234375,
      "loss": 0.0589,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.457432508468628,
      "rewards/margins": 8.91236686706543,
      "rewards/rejected": -11.369799613952637,
      "step": 1065
    },
    {
      "epoch": 2.2816,
      "grad_norm": 1.9213427305221558,
      "learning_rate": 7.20060672845091e-07,
      "logits/chosen": -1.6789592504501343,
      "logits/rejected": -1.6316735744476318,
      "logps/chosen": -226.3343505859375,
      "logps/rejected": -381.14398193359375,
      "loss": 0.0501,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.6402697563171387,
      "rewards/margins": 9.172629356384277,
      "rewards/rejected": -11.812898635864258,
      "step": 1070
    },
    {
      "epoch": 2.2922666666666665,
      "grad_norm": 2.5269217491149902,
      "learning_rate": 6.99962512238733e-07,
      "logits/chosen": -1.657263994216919,
      "logits/rejected": -1.5200378894805908,
      "logps/chosen": -226.5382537841797,
      "logps/rejected": -372.1704406738281,
      "loss": 0.1022,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.4905364513397217,
      "rewards/margins": 8.78575325012207,
      "rewards/rejected": -11.276289939880371,
      "step": 1075
    },
    {
      "epoch": 2.3029333333333333,
      "grad_norm": 2.181729555130005,
      "learning_rate": 6.801030710594949e-07,
      "logits/chosen": -1.628305435180664,
      "logits/rejected": -1.5248959064483643,
      "logps/chosen": -246.23812866210938,
      "logps/rejected": -402.61614990234375,
      "loss": 0.0546,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.856438159942627,
      "rewards/margins": 9.355535507202148,
      "rewards/rejected": -12.211973190307617,
      "step": 1080
    },
    {
      "epoch": 2.3136,
      "grad_norm": 1.0526963472366333,
      "learning_rate": 6.604849830494264e-07,
      "logits/chosen": -1.700221061706543,
      "logits/rejected": -1.608945608139038,
      "logps/chosen": -239.19741821289062,
      "logps/rejected": -369.9862976074219,
      "loss": 0.0432,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.78136944770813,
      "rewards/margins": 8.838476181030273,
      "rewards/rejected": -11.619847297668457,
      "step": 1085
    },
    {
      "epoch": 2.3242666666666665,
      "grad_norm": 0.7994614839553833,
      "learning_rate": 6.411108499425253e-07,
      "logits/chosen": -1.7244764566421509,
      "logits/rejected": -1.5778989791870117,
      "logps/chosen": -234.7858123779297,
      "logps/rejected": -374.0656433105469,
      "loss": 0.0539,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.5700016021728516,
      "rewards/margins": 9.354166984558105,
      "rewards/rejected": -11.924168586730957,
      "step": 1090
    },
    {
      "epoch": 2.3349333333333333,
      "grad_norm": 0.7055251598358154,
      "learning_rate": 6.219832411197013e-07,
      "logits/chosen": -1.678105354309082,
      "logits/rejected": -1.5513595342636108,
      "logps/chosen": -240.69384765625,
      "logps/rejected": -375.53179931640625,
      "loss": 0.0666,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.769808053970337,
      "rewards/margins": 9.200688362121582,
      "rewards/rejected": -11.970497131347656,
      "step": 1095
    },
    {
      "epoch": 2.3456,
      "grad_norm": 0.7503064870834351,
      "learning_rate": 6.031046932680229e-07,
      "logits/chosen": -1.69338059425354,
      "logits/rejected": -1.5614005327224731,
      "logps/chosen": -256.23828125,
      "logps/rejected": -392.1869201660156,
      "loss": 0.0492,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.936000108718872,
      "rewards/margins": 9.314740180969238,
      "rewards/rejected": -12.250740051269531,
      "step": 1100
    },
    {
      "epoch": 2.3562666666666665,
      "grad_norm": 0.8023340702056885,
      "learning_rate": 5.844777100443091e-07,
      "logits/chosen": -1.7291017770767212,
      "logits/rejected": -1.5592310428619385,
      "logps/chosen": -237.56771850585938,
      "logps/rejected": -390.6042785644531,
      "loss": 0.062,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.023751735687256,
      "rewards/margins": 9.082342147827148,
      "rewards/rejected": -12.106094360351562,
      "step": 1105
    },
    {
      "epoch": 2.3669333333333333,
      "grad_norm": 0.3595770299434662,
      "learning_rate": 5.66104761743092e-07,
      "logits/chosen": -1.7586097717285156,
      "logits/rejected": -1.6678698062896729,
      "logps/chosen": -252.323974609375,
      "logps/rejected": -401.20147705078125,
      "loss": 0.042,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.7489242553710938,
      "rewards/margins": 9.122079849243164,
      "rewards/rejected": -12.871004104614258,
      "step": 1110
    },
    {
      "epoch": 2.3776,
      "grad_norm": 3.4164505004882812,
      "learning_rate": 5.479882849690102e-07,
      "logits/chosen": -1.732012391090393,
      "logits/rejected": -1.5598955154418945,
      "logps/chosen": -234.59298706054688,
      "logps/rejected": -378.66162109375,
      "loss": 0.0873,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.783665895462036,
      "rewards/margins": 9.570892333984375,
      "rewards/rejected": -12.354558944702148,
      "step": 1115
    },
    {
      "epoch": 2.3882666666666665,
      "grad_norm": 1.439928650856018,
      "learning_rate": 5.301306823136687e-07,
      "logits/chosen": -1.664751410484314,
      "logits/rejected": -1.590052604675293,
      "logps/chosen": -214.80111694335938,
      "logps/rejected": -394.08221435546875,
      "loss": 0.0645,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2062828540802,
      "rewards/margins": 9.442559242248535,
      "rewards/rejected": -11.648841857910156,
      "step": 1120
    },
    {
      "epoch": 2.3989333333333334,
      "grad_norm": 1.5526204109191895,
      "learning_rate": 5.125343220370088e-07,
      "logits/chosen": -1.6302855014801025,
      "logits/rejected": -1.553674578666687,
      "logps/chosen": -257.54541015625,
      "logps/rejected": -374.77142333984375,
      "loss": 0.0523,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.0847949981689453,
      "rewards/margins": 8.612579345703125,
      "rewards/rejected": -11.69737434387207,
      "step": 1125
    },
    {
      "epoch": 2.4096,
      "grad_norm": 0.7676222324371338,
      "learning_rate": 4.952015377532329e-07,
      "logits/chosen": -1.8063156604766846,
      "logits/rejected": -1.6408159732818604,
      "logps/chosen": -240.57730102539062,
      "logps/rejected": -386.2501220703125,
      "loss": 0.0612,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.309523820877075,
      "rewards/margins": 9.168109893798828,
      "rewards/rejected": -12.47763442993164,
      "step": 1130
    },
    {
      "epoch": 2.4202666666666666,
      "grad_norm": 2.877019166946411,
      "learning_rate": 4.781346281213204e-07,
      "logits/chosen": -1.8087190389633179,
      "logits/rejected": -1.6685491800308228,
      "logps/chosen": -243.5975799560547,
      "logps/rejected": -416.5228576660156,
      "loss": 0.0539,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.3564605712890625,
      "rewards/margins": 9.94178581237793,
      "rewards/rejected": -13.298245429992676,
      "step": 1135
    },
    {
      "epoch": 2.4309333333333334,
      "grad_norm": 3.171461582183838,
      "learning_rate": 4.613358565401818e-07,
      "logits/chosen": -1.6455013751983643,
      "logits/rejected": -1.5212013721466064,
      "logps/chosen": -240.3773193359375,
      "logps/rejected": -380.5497131347656,
      "loss": 0.076,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.7457993030548096,
      "rewards/margins": 8.988884925842285,
      "rewards/rejected": -11.7346830368042,
      "step": 1140
    },
    {
      "epoch": 2.4416,
      "grad_norm": 0.8925107717514038,
      "learning_rate": 4.448074508484906e-07,
      "logits/chosen": -1.6455951929092407,
      "logits/rejected": -1.4987901449203491,
      "logps/chosen": -257.70379638671875,
      "logps/rejected": -374.9892272949219,
      "loss": 0.0572,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.067699909210205,
      "rewards/margins": 9.026286125183105,
      "rewards/rejected": -12.093984603881836,
      "step": 1145
    },
    {
      "epoch": 2.4522666666666666,
      "grad_norm": 1.1913326978683472,
      "learning_rate": 4.2855160302922777e-07,
      "logits/chosen": -1.690572738647461,
      "logits/rejected": -1.6463066339492798,
      "logps/chosen": -234.546630859375,
      "logps/rejected": -398.32763671875,
      "loss": 0.0459,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.1086277961730957,
      "rewards/margins": 9.187312126159668,
      "rewards/rejected": -12.295941352844238,
      "step": 1150
    },
    {
      "epoch": 2.4629333333333334,
      "grad_norm": 2.599015235900879,
      "learning_rate": 4.125704689189819e-07,
      "logits/chosen": -1.5713093280792236,
      "logits/rejected": -1.5037789344787598,
      "logps/chosen": -241.7811737060547,
      "logps/rejected": -394.9588623046875,
      "loss": 0.066,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8472790718078613,
      "rewards/margins": 9.292223930358887,
      "rewards/rejected": -12.139504432678223,
      "step": 1155
    },
    {
      "epoch": 2.4736000000000002,
      "grad_norm": 3.4746813774108887,
      "learning_rate": 3.9686616792204677e-07,
      "logits/chosen": -1.671574354171753,
      "logits/rejected": -1.6419601440429688,
      "logps/chosen": -243.90811157226562,
      "logps/rejected": -380.66265869140625,
      "loss": 0.0644,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -2.9403159618377686,
      "rewards/margins": 8.807522773742676,
      "rewards/rejected": -11.747838020324707,
      "step": 1160
    },
    {
      "epoch": 2.4842666666666666,
      "grad_norm": 1.217083215713501,
      "learning_rate": 3.8144078272934676e-07,
      "logits/chosen": -1.616916298866272,
      "logits/rejected": -1.5705225467681885,
      "logps/chosen": -246.2116241455078,
      "logps/rejected": -400.27520751953125,
      "loss": 0.054,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.046478748321533,
      "rewards/margins": 9.513192176818848,
      "rewards/rejected": -12.559671401977539,
      "step": 1165
    },
    {
      "epoch": 2.4949333333333334,
      "grad_norm": 0.5213105082511902,
      "learning_rate": 3.6629635904222943e-07,
      "logits/chosen": -1.632601022720337,
      "logits/rejected": -1.580075740814209,
      "logps/chosen": -242.1397705078125,
      "logps/rejected": -392.51708984375,
      "loss": 0.035,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.779862642288208,
      "rewards/margins": 9.45389461517334,
      "rewards/rejected": -12.233755111694336,
      "step": 1170
    },
    {
      "epoch": 2.5056000000000003,
      "grad_norm": 4.385895729064941,
      "learning_rate": 3.514349053011698e-07,
      "logits/chosen": -1.5838334560394287,
      "logits/rejected": -1.48439621925354,
      "logps/chosen": -240.0968475341797,
      "logps/rejected": -380.6497802734375,
      "loss": 0.0528,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.24969482421875,
      "rewards/margins": 9.019821166992188,
      "rewards/rejected": -12.269515991210938,
      "step": 1175
    },
    {
      "epoch": 2.5162666666666667,
      "grad_norm": 1.1202504634857178,
      "learning_rate": 3.3685839241941133e-07,
      "logits/chosen": -1.6621793508529663,
      "logits/rejected": -1.585744857788086,
      "logps/chosen": -241.1162872314453,
      "logps/rejected": -379.06610107421875,
      "loss": 0.053,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.787099838256836,
      "rewards/margins": 9.54288101196289,
      "rewards/rejected": -12.329981803894043,
      "step": 1180
    },
    {
      "epoch": 2.5269333333333335,
      "grad_norm": 0.898692786693573,
      "learning_rate": 3.2256875352158435e-07,
      "logits/chosen": -1.6413805484771729,
      "logits/rejected": -1.5792820453643799,
      "logps/chosen": -241.3943634033203,
      "logps/rejected": -397.4687805175781,
      "loss": 0.0829,
      "rewards/accuracies": 0.96875,
      "rewards/chosen": -3.0518360137939453,
      "rewards/margins": 9.231449127197266,
      "rewards/rejected": -12.283285140991211,
      "step": 1185
    },
    {
      "epoch": 2.5376,
      "grad_norm": 1.140761375427246,
      "learning_rate": 3.085678836873399e-07,
      "logits/chosen": -1.6138441562652588,
      "logits/rejected": -1.5633118152618408,
      "logps/chosen": -232.3341827392578,
      "logps/rejected": -381.39111328125,
      "loss": 0.0851,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.112748146057129,
      "rewards/margins": 8.837594985961914,
      "rewards/rejected": -11.95034408569336,
      "step": 1190
    },
    {
      "epoch": 2.5482666666666667,
      "grad_norm": 4.43955135345459,
      "learning_rate": 2.9485763970002106e-07,
      "logits/chosen": -1.7343018054962158,
      "logits/rejected": -1.5757277011871338,
      "logps/chosen": -242.49642944335938,
      "logps/rejected": -405.9391784667969,
      "loss": 0.0528,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.864837408065796,
      "rewards/margins": 9.605855941772461,
      "rewards/rejected": -12.470693588256836,
      "step": 1195
    },
    {
      "epoch": 2.558933333333333,
      "grad_norm": 0.7420125603675842,
      "learning_rate": 2.8143983980042174e-07,
      "logits/chosen": -1.6621475219726562,
      "logits/rejected": -1.5564953088760376,
      "logps/chosen": -231.9013214111328,
      "logps/rejected": -376.8731689453125,
      "loss": 0.0673,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.134626865386963,
      "rewards/margins": 8.874670028686523,
      "rewards/rejected": -12.009298324584961,
      "step": 1200
    },
    {
      "epoch": 2.5696,
      "grad_norm": 0.8060418963432312,
      "learning_rate": 2.683162634456515e-07,
      "logits/chosen": -1.6644542217254639,
      "logits/rejected": -1.5664650201797485,
      "logps/chosen": -258.3770751953125,
      "logps/rejected": -404.95135498046875,
      "loss": 0.0543,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.6065478324890137,
      "rewards/margins": 9.685090065002441,
      "rewards/rejected": -12.29163646697998,
      "step": 1205
    },
    {
      "epoch": 2.5802666666666667,
      "grad_norm": 0.8783431053161621,
      "learning_rate": 2.5548865107314606e-07,
      "logits/chosen": -1.6931167840957642,
      "logits/rejected": -1.5787171125411987,
      "logps/chosen": -248.09957885742188,
      "logps/rejected": -387.348388671875,
      "loss": 0.0508,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2612154483795166,
      "rewards/margins": 10.170672416687012,
      "rewards/rejected": -12.431889533996582,
      "step": 1210
    },
    {
      "epoch": 2.590933333333333,
      "grad_norm": 2.2933311462402344,
      "learning_rate": 2.429587038698511e-07,
      "logits/chosen": -1.5951554775238037,
      "logits/rejected": -1.5162432193756104,
      "logps/chosen": -224.2011260986328,
      "logps/rejected": -361.96673583984375,
      "loss": 0.0685,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.2784221172332764,
      "rewards/margins": 8.822007179260254,
      "rewards/rejected": -12.100428581237793,
      "step": 1215
    },
    {
      "epoch": 2.6016,
      "grad_norm": 0.8401004076004028,
      "learning_rate": 2.307280835466144e-07,
      "logits/chosen": -1.6584374904632568,
      "logits/rejected": -1.5249509811401367,
      "logps/chosen": -243.69009399414062,
      "logps/rejected": -373.2393798828125,
      "loss": 0.068,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.3214409351348877,
      "rewards/margins": 9.552881240844727,
      "rewards/rejected": -12.874322891235352,
      "step": 1220
    },
    {
      "epoch": 2.6122666666666667,
      "grad_norm": 2.254423141479492,
      "learning_rate": 2.1879841211780856e-07,
      "logits/chosen": -1.6165263652801514,
      "logits/rejected": -1.5144685506820679,
      "logps/chosen": -231.9131317138672,
      "logps/rejected": -352.79345703125,
      "loss": 0.0671,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.164689302444458,
      "rewards/margins": 8.177474975585938,
      "rewards/rejected": -11.342164993286133,
      "step": 1225
    },
    {
      "epoch": 2.622933333333333,
      "grad_norm": 4.380727767944336,
      "learning_rate": 2.0717127168622202e-07,
      "logits/chosen": -1.6678218841552734,
      "logits/rejected": -1.5193488597869873,
      "logps/chosen": -248.5584259033203,
      "logps/rejected": -370.3858947753906,
      "loss": 0.0618,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.2485344409942627,
      "rewards/margins": 9.039493560791016,
      "rewards/rejected": -12.2880277633667,
      "step": 1230
    },
    {
      "epoch": 2.6336,
      "grad_norm": 0.5853209495544434,
      "learning_rate": 1.958482042332424e-07,
      "logits/chosen": -1.6132605075836182,
      "logits/rejected": -1.4864953756332397,
      "logps/chosen": -249.97372436523438,
      "logps/rejected": -382.3087463378906,
      "loss": 0.0471,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.8786864280700684,
      "rewards/margins": 9.31507396697998,
      "rewards/rejected": -12.193760871887207,
      "step": 1235
    },
    {
      "epoch": 2.6442666666666668,
      "grad_norm": 0.6446689367294312,
      "learning_rate": 1.8483071141435937e-07,
      "logits/chosen": -1.5566718578338623,
      "logits/rejected": -1.482312560081482,
      "logps/chosen": -243.97195434570312,
      "logps/rejected": -386.7613830566406,
      "loss": 0.0738,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.7106494903564453,
      "rewards/margins": 9.434020042419434,
      "rewards/rejected": -12.144668579101562,
      "step": 1240
    },
    {
      "epoch": 2.654933333333333,
      "grad_norm": 4.565989017486572,
      "learning_rate": 1.741202543600176e-07,
      "logits/chosen": -1.6722030639648438,
      "logits/rejected": -1.593416452407837,
      "logps/chosen": -229.83444213867188,
      "logps/rejected": -398.0259704589844,
      "loss": 0.0547,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.4450182914733887,
      "rewards/margins": 9.658830642700195,
      "rewards/rejected": -13.103849411010742,
      "step": 1245
    },
    {
      "epoch": 2.6656,
      "grad_norm": 0.8811970949172974,
      "learning_rate": 1.6371825348184155e-07,
      "logits/chosen": -1.6740272045135498,
      "logits/rejected": -1.5639140605926514,
      "logps/chosen": -237.81863403320312,
      "logps/rejected": -390.57489013671875,
      "loss": 0.0788,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.0801661014556885,
      "rewards/margins": 8.766717910766602,
      "rewards/rejected": -11.846883773803711,
      "step": 1250
    },
    {
      "epoch": 2.676266666666667,
      "grad_norm": 0.733801007270813,
      "learning_rate": 1.5362608828426246e-07,
      "logits/chosen": -1.6724077463150024,
      "logits/rejected": -1.5936537981033325,
      "logps/chosen": -244.7788848876953,
      "logps/rejected": -397.16827392578125,
      "loss": 0.06,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.0939319133758545,
      "rewards/margins": 9.266348838806152,
      "rewards/rejected": -12.360280990600586,
      "step": 1255
    },
    {
      "epoch": 2.686933333333333,
      "grad_norm": 1.226430892944336,
      "learning_rate": 1.438450971815686e-07,
      "logits/chosen": -1.6890594959259033,
      "logits/rejected": -1.644351601600647,
      "logps/chosen": -225.499267578125,
      "logps/rejected": -397.11492919921875,
      "loss": 0.0462,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.3902297019958496,
      "rewards/margins": 9.87806510925293,
      "rewards/rejected": -13.268295288085938,
      "step": 1260
    },
    {
      "epoch": 2.6976,
      "grad_norm": 1.1510404348373413,
      "learning_rate": 1.3437657732040783e-07,
      "logits/chosen": -1.6579421758651733,
      "logits/rejected": -1.5118043422698975,
      "logps/chosen": -248.87814331054688,
      "logps/rejected": -364.0889587402344,
      "loss": 0.0622,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.0955281257629395,
      "rewards/margins": 8.875585556030273,
      "rewards/rejected": -11.971113204956055,
      "step": 1265
    },
    {
      "epoch": 2.708266666666667,
      "grad_norm": 0.9455292820930481,
      "learning_rate": 1.2522178440776022e-07,
      "logits/chosen": -1.6150785684585571,
      "logits/rejected": -1.5976345539093018,
      "logps/chosen": -249.5107421875,
      "logps/rejected": -389.8973083496094,
      "loss": 0.0803,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.392077922821045,
      "rewards/margins": 9.01916217803955,
      "rewards/rejected": -12.41124153137207,
      "step": 1270
    },
    {
      "epoch": 2.718933333333333,
      "grad_norm": 1.740357756614685,
      "learning_rate": 1.1638193254440655e-07,
      "logits/chosen": -1.6622123718261719,
      "logits/rejected": -1.534313678741455,
      "logps/chosen": -238.5977325439453,
      "logps/rejected": -380.42108154296875,
      "loss": 0.0555,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.4845123291015625,
      "rewards/margins": 9.121153831481934,
      "rewards/rejected": -12.60566520690918,
      "step": 1275
    },
    {
      "epoch": 2.7296,
      "grad_norm": 1.714776635169983,
      "learning_rate": 1.0785819406391562e-07,
      "logits/chosen": -1.5962209701538086,
      "logits/rejected": -1.5493957996368408,
      "logps/chosen": -271.0741271972656,
      "logps/rejected": -369.3987731933594,
      "loss": 0.042,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.6246178150177,
      "rewards/margins": 9.22441577911377,
      "rewards/rejected": -11.849031448364258,
      "step": 1280
    },
    {
      "epoch": 2.740266666666667,
      "grad_norm": 2.9277751445770264,
      "learning_rate": 9.965169937717095e-08,
      "logits/chosen": -1.64768385887146,
      "logits/rejected": -1.5789892673492432,
      "logps/chosen": -234.07608032226562,
      "logps/rejected": -388.78961181640625,
      "loss": 0.0615,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.3321361541748047,
      "rewards/margins": 9.532690048217773,
      "rewards/rejected": -12.864827156066895,
      "step": 1285
    },
    {
      "epoch": 2.7509333333333332,
      "grad_norm": 4.05463981628418,
      "learning_rate": 9.176353682245676e-08,
      "logits/chosen": -1.7111181020736694,
      "logits/rejected": -1.5939412117004395,
      "logps/chosen": -242.18759155273438,
      "logps/rejected": -376.65948486328125,
      "loss": 0.0931,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.3087191581726074,
      "rewards/margins": 9.18929672241211,
      "rewards/rejected": -12.498015403747559,
      "step": 1290
    },
    {
      "epoch": 2.7616,
      "grad_norm": 4.355930805206299,
      "learning_rate": 8.41947525211223e-08,
      "logits/chosen": -1.528749704360962,
      "logits/rejected": -1.5081889629364014,
      "logps/chosen": -232.96383666992188,
      "logps/rejected": -353.97906494140625,
      "loss": 0.1119,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.941885471343994,
      "rewards/margins": 8.040489196777344,
      "rewards/rejected": -10.98237419128418,
      "step": 1295
    },
    {
      "epoch": 2.772266666666667,
      "grad_norm": 1.2377660274505615,
      "learning_rate": 7.694635023884789e-08,
      "logits/chosen": -1.596066951751709,
      "logits/rejected": -1.5349980592727661,
      "logps/chosen": -244.0199737548828,
      "logps/rejected": -365.12384033203125,
      "loss": 0.0548,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.9959444999694824,
      "rewards/margins": 8.654120445251465,
      "rewards/rejected": -11.650065422058105,
      "step": 1300
    },
    {
      "epoch": 2.7829333333333333,
      "grad_norm": 4.812592506408691,
      "learning_rate": 7.00192912525241e-08,
      "logits/chosen": -1.6420648097991943,
      "logits/rejected": -1.5604394674301147,
      "logps/chosen": -229.23388671875,
      "logps/rejected": -394.1202697753906,
      "loss": 0.0746,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -3.189704418182373,
      "rewards/margins": 9.603094100952148,
      "rewards/rejected": -12.79279899597168,
      "step": 1305
    },
    {
      "epoch": 2.7936,
      "grad_norm": 2.0514466762542725,
      "learning_rate": 6.341449422277047e-08,
      "logits/chosen": -1.5872209072113037,
      "logits/rejected": -1.4994614124298096,
      "logps/chosen": -246.319091796875,
      "logps/rejected": -378.219970703125,
      "loss": 0.0499,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.6841087341308594,
      "rewards/margins": 10.14326286315918,
      "rewards/rejected": -12.827372550964355,
      "step": 1310
    },
    {
      "epoch": 2.804266666666667,
      "grad_norm": 3.7857437133789062,
      "learning_rate": 5.7132835072101486e-08,
      "logits/chosen": -1.6783745288848877,
      "logits/rejected": -1.5917742252349854,
      "logps/chosen": -280.02191162109375,
      "logps/rejected": -345.45831298828125,
      "loss": 0.0696,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.7913777828216553,
      "rewards/margins": 8.280149459838867,
      "rewards/rejected": -11.071527481079102,
      "step": 1315
    },
    {
      "epoch": 2.8149333333333333,
      "grad_norm": 2.1288628578186035,
      "learning_rate": 5.117514686876379e-08,
      "logits/chosen": -1.5824289321899414,
      "logits/rejected": -1.4594101905822754,
      "logps/chosen": -239.4272003173828,
      "logps/rejected": -340.62200927734375,
      "loss": 0.0832,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9561493396759033,
      "rewards/margins": 7.562721252441406,
      "rewards/rejected": -10.51887035369873,
      "step": 1320
    },
    {
      "epoch": 2.8256,
      "grad_norm": 5.216054439544678,
      "learning_rate": 4.5542219716255274e-08,
      "logits/chosen": -1.6920030117034912,
      "logits/rejected": -1.5826241970062256,
      "logps/chosen": -247.36904907226562,
      "logps/rejected": -388.71051025390625,
      "loss": 0.0521,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.8860912322998047,
      "rewards/margins": 9.873689651489258,
      "rewards/rejected": -12.75977897644043,
      "step": 1325
    },
    {
      "epoch": 2.836266666666667,
      "grad_norm": 1.0527479648590088,
      "learning_rate": 4.023480064854174e-08,
      "logits/chosen": -1.712881088256836,
      "logits/rejected": -1.6000264883041382,
      "logps/chosen": -236.73312377929688,
      "logps/rejected": -387.08599853515625,
      "loss": 0.0456,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.8067479133605957,
      "rewards/margins": 10.028037071228027,
      "rewards/rejected": -12.834785461425781,
      "step": 1330
    },
    {
      "epoch": 2.8469333333333333,
      "grad_norm": 3.546607494354248,
      "learning_rate": 3.525359353098584e-08,
      "logits/chosen": -1.6619724035263062,
      "logits/rejected": -1.5283750295639038,
      "logps/chosen": -236.0811309814453,
      "logps/rejected": -376.9278259277344,
      "loss": 0.0546,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.1373705863952637,
      "rewards/margins": 9.576279640197754,
      "rewards/rejected": -12.713650703430176,
      "step": 1335
    },
    {
      "epoch": 2.8576,
      "grad_norm": 1.9862865209579468,
      "learning_rate": 3.059925896700094e-08,
      "logits/chosen": -1.5844295024871826,
      "logits/rejected": -1.4807649850845337,
      "logps/chosen": -267.7585144042969,
      "logps/rejected": -370.53790283203125,
      "loss": 0.0592,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.931180477142334,
      "rewards/margins": 8.453359603881836,
      "rewards/rejected": -11.384538650512695,
      "step": 1340
    },
    {
      "epoch": 2.8682666666666665,
      "grad_norm": 3.8812670707702637,
      "learning_rate": 2.6272414210443964e-08,
      "logits/chosen": -1.6309881210327148,
      "logits/rejected": -1.5491442680358887,
      "logps/chosen": -225.6214599609375,
      "logps/rejected": -383.81243896484375,
      "loss": 0.0824,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.9007303714752197,
      "rewards/margins": 9.299989700317383,
      "rewards/rejected": -12.200719833374023,
      "step": 1345
    },
    {
      "epoch": 2.8789333333333333,
      "grad_norm": 4.220300197601318,
      "learning_rate": 2.2273633083753632e-08,
      "logits/chosen": -1.6484569311141968,
      "logits/rejected": -1.5259983539581299,
      "logps/chosen": -221.6566619873047,
      "logps/rejected": -352.6129455566406,
      "loss": 0.0606,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.176244020462036,
      "rewards/margins": 8.576908111572266,
      "rewards/rejected": -11.753152847290039,
      "step": 1350
    },
    {
      "epoch": 2.8895999999999997,
      "grad_norm": 1.0096988677978516,
      "learning_rate": 1.8603445901851647e-08,
      "logits/chosen": -1.6377588510513306,
      "logits/rejected": -1.559996485710144,
      "logps/chosen": -252.84579467773438,
      "logps/rejected": -386.17755126953125,
      "loss": 0.0955,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.2294230461120605,
      "rewards/margins": 8.582195281982422,
      "rewards/rejected": -11.811617851257324,
      "step": 1355
    },
    {
      "epoch": 2.9002666666666665,
      "grad_norm": 1.8424628973007202,
      "learning_rate": 1.5262339401813375e-08,
      "logits/chosen": -1.7368948459625244,
      "logits/rejected": -1.6223742961883545,
      "logps/chosen": -211.1990203857422,
      "logps/rejected": -368.0191650390625,
      "loss": 0.0704,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.6657638549804688,
      "rewards/margins": 9.301887512207031,
      "rewards/rejected": -11.9676513671875,
      "step": 1360
    },
    {
      "epoch": 2.9109333333333334,
      "grad_norm": 3.6020452976226807,
      "learning_rate": 1.2250756678315601e-08,
      "logits/chosen": -1.6663650274276733,
      "logits/rejected": -1.57073974609375,
      "logps/chosen": -233.00131225585938,
      "logps/rejected": -400.0315856933594,
      "loss": 0.0579,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.8992886543273926,
      "rewards/margins": 10.07255744934082,
      "rewards/rejected": -12.971845626831055,
      "step": 1365
    },
    {
      "epoch": 2.9215999999999998,
      "grad_norm": 1.2735259532928467,
      "learning_rate": 9.569097124875193e-09,
      "logits/chosen": -1.6023895740509033,
      "logits/rejected": -1.4932624101638794,
      "logps/chosen": -224.2072296142578,
      "logps/rejected": -374.86883544921875,
      "loss": 0.0506,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.029714584350586,
      "rewards/margins": 9.75147533416748,
      "rewards/rejected": -12.781190872192383,
      "step": 1370
    },
    {
      "epoch": 2.9322666666666666,
      "grad_norm": 3.791727066040039,
      "learning_rate": 7.217716380881479e-09,
      "logits/chosen": -1.6325794458389282,
      "logits/rejected": -1.6049026250839233,
      "logps/chosen": -258.8813171386719,
      "logps/rejected": -384.8707580566406,
      "loss": 0.0517,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -3.0506906509399414,
      "rewards/margins": 9.173063278198242,
      "rewards/rejected": -12.223753929138184,
      "step": 1375
    },
    {
      "epoch": 2.9429333333333334,
      "grad_norm": 4.857158660888672,
      "learning_rate": 5.196926284430359e-09,
      "logits/chosen": -1.6092033386230469,
      "logits/rejected": -1.5137813091278076,
      "logps/chosen": -244.2127685546875,
      "logps/rejected": -382.6106872558594,
      "loss": 0.0808,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.1281540393829346,
      "rewards/margins": 9.14082145690918,
      "rewards/rejected": -12.268976211547852,
      "step": 1380
    },
    {
      "epoch": 2.9536,
      "grad_norm": 3.6009581089019775,
      "learning_rate": 3.5069948309712775e-09,
      "logits/chosen": -1.5844300985336304,
      "logits/rejected": -1.5796252489089966,
      "logps/chosen": -220.5863800048828,
      "logps/rejected": -371.11956787109375,
      "loss": 0.0649,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.970752716064453,
      "rewards/margins": 8.276488304138184,
      "rewards/rejected": -11.247241020202637,
      "step": 1385
    },
    {
      "epoch": 2.9642666666666666,
      "grad_norm": 3.476202964782715,
      "learning_rate": 2.1481461377634294e-09,
      "logits/chosen": -1.631340742111206,
      "logits/rejected": -1.5940053462982178,
      "logps/chosen": -231.7400360107422,
      "logps/rejected": -374.5093078613281,
      "loss": 0.0465,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -3.1524415016174316,
      "rewards/margins": 9.017885208129883,
      "rewards/rejected": -12.170326232910156,
      "step": 1390
    },
    {
      "epoch": 2.9749333333333334,
      "grad_norm": 1.2247248888015747,
      "learning_rate": 1.120560414154259e-09,
      "logits/chosen": -1.642568588256836,
      "logits/rejected": -1.479210615158081,
      "logps/chosen": -236.7920379638672,
      "logps/rejected": -384.9251708984375,
      "loss": 0.0431,
      "rewards/accuracies": 0.9937499761581421,
      "rewards/chosen": -2.9622087478637695,
      "rewards/margins": 9.579465866088867,
      "rewards/rejected": -12.541674613952637,
      "step": 1395
    },
    {
      "epoch": 2.9856,
      "grad_norm": 1.0781242847442627,
      "learning_rate": 4.2437393768079983e-10,
      "logits/chosen": -1.670657753944397,
      "logits/rejected": -1.5803184509277344,
      "logps/chosen": -252.31094360351562,
      "logps/rejected": -374.4371032714844,
      "loss": 0.0526,
      "rewards/accuracies": 0.981249988079071,
      "rewards/chosen": -2.9776816368103027,
      "rewards/margins": 8.582575798034668,
      "rewards/rejected": -11.560258865356445,
      "step": 1400
    },
    {
      "epoch": 2.9962666666666666,
      "grad_norm": 1.7229729890823364,
      "learning_rate": 5.967903599635171e-11,
      "logits/chosen": -1.6550830602645874,
      "logits/rejected": -1.4953168630599976,
      "logps/chosen": -239.16610717773438,
      "logps/rejected": -392.11444091796875,
      "loss": 0.0611,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.644193172454834,
      "rewards/margins": 9.434123039245605,
      "rewards/rejected": -12.078317642211914,
      "step": 1405
    }
  ],
  "logging_steps": 5,
  "max_steps": 1407,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}