{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 20.0,
  "eval_steps": 500,
  "global_step": 580,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.3448275862068966,
      "grad_norm": 3.9107234477996826,
      "learning_rate": 8.620689655172415e-07,
      "logits/chosen": -2.363854169845581,
      "logits/rejected": -2.338671922683716,
      "logps/chosen": -76.8819351196289,
      "logps/rejected": -75.86869049072266,
      "loss": 0.6942,
      "rewards/accuracies": 0.375,
      "rewards/chosen": -0.003996999468654394,
      "rewards/margins": -0.004145228303968906,
      "rewards/rejected": 0.00014822949015069753,
      "step": 10
    },
    {
      "epoch": 0.6896551724137931,
      "grad_norm": 4.328212738037109,
      "learning_rate": 1.724137931034483e-06,
      "logits/chosen": -2.3617663383483887,
      "logits/rejected": -2.3477540016174316,
      "logps/chosen": -93.76543426513672,
      "logps/rejected": -75.16656494140625,
      "loss": 0.6931,
      "rewards/accuracies": 0.48750001192092896,
      "rewards/chosen": 0.002653368515893817,
      "rewards/margins": 0.006352785974740982,
      "rewards/rejected": -0.003699416993185878,
      "step": 20
    },
    {
      "epoch": 1.0344827586206897,
      "grad_norm": 3.473827838897705,
      "learning_rate": 2.5862068965517246e-06,
      "logits/chosen": -2.3354058265686035,
      "logits/rejected": -2.3330740928649902,
      "logps/chosen": -138.9960479736328,
      "logps/rejected": -67.00438690185547,
      "loss": 0.6927,
      "rewards/accuracies": 0.44999998807907104,
      "rewards/chosen": 0.01745142787694931,
      "rewards/margins": 0.013760591857135296,
      "rewards/rejected": 0.0036908381152898073,
      "step": 30
    },
    {
      "epoch": 1.3793103448275863,
      "grad_norm": 3.9458820819854736,
      "learning_rate": 3.448275862068966e-06,
      "logits/chosen": -2.3478751182556152,
      "logits/rejected": -2.3700289726257324,
      "logps/chosen": -75.59983825683594,
      "logps/rejected": -67.20357513427734,
      "loss": 0.69,
      "rewards/accuracies": 0.5,
      "rewards/chosen": 0.00024204826331697404,
      "rewards/margins": 0.005866119172424078,
      "rewards/rejected": -0.005624071694910526,
      "step": 40
    },
    {
      "epoch": 1.7241379310344827,
      "grad_norm": 4.299890041351318,
      "learning_rate": 4.310344827586207e-06,
      "logits/chosen": -2.338815212249756,
      "logits/rejected": -2.3532769680023193,
      "logps/chosen": -72.74342346191406,
      "logps/rejected": -87.63409423828125,
      "loss": 0.6899,
      "rewards/accuracies": 0.574999988079071,
      "rewards/chosen": -0.003249581903219223,
      "rewards/margins": 0.006888723932206631,
      "rewards/rejected": -0.010138307698071003,
      "step": 50
    },
    {
      "epoch": 2.0689655172413794,
      "grad_norm": 4.957555294036865,
      "learning_rate": 4.999818897894192e-06,
      "logits/chosen": -2.344572067260742,
      "logits/rejected": -2.345888614654541,
      "logps/chosen": -80.16046142578125,
      "logps/rejected": -73.05986022949219,
      "loss": 0.6802,
      "rewards/accuracies": 0.637499988079071,
      "rewards/chosen": -0.013288321904838085,
      "rewards/margins": 0.020968889817595482,
      "rewards/rejected": -0.03425721079111099,
      "step": 60
    },
    {
      "epoch": 2.413793103448276,
      "grad_norm": 4.212078094482422,
      "learning_rate": 4.9934830787948756e-06,
      "logits/chosen": -2.3330090045928955,
      "logits/rejected": -2.328575372695923,
      "logps/chosen": -73.63365173339844,
      "logps/rejected": -74.45356750488281,
      "loss": 0.6693,
      "rewards/accuracies": 0.7124999761581421,
      "rewards/chosen": -0.009874681942164898,
      "rewards/margins": 0.04785541445016861,
      "rewards/rejected": -0.05773010104894638,
      "step": 70
    },
    {
      "epoch": 2.7586206896551726,
      "grad_norm": 4.446742534637451,
      "learning_rate": 4.978118375700895e-06,
      "logits/chosen": -2.3472495079040527,
      "logits/rejected": -2.3697409629821777,
      "logps/chosen": -73.44490814208984,
      "logps/rejected": -89.69837951660156,
      "loss": 0.6553,
      "rewards/accuracies": 0.8500000238418579,
      "rewards/chosen": -0.022817375138401985,
      "rewards/margins": 0.09045850485563278,
      "rewards/rejected": -0.11327588558197021,
      "step": 80
    },
    {
      "epoch": 3.103448275862069,
      "grad_norm": 4.435715198516846,
      "learning_rate": 4.953780424089803e-06,
      "logits/chosen": -2.346717596054077,
      "logits/rejected": -2.3626608848571777,
      "logps/chosen": -85.64796447753906,
      "logps/rejected": -80.06268310546875,
      "loss": 0.6432,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.0543200746178627,
      "rewards/margins": 0.1085677295923233,
      "rewards/rejected": -0.1628878116607666,
      "step": 90
    },
    {
      "epoch": 3.4482758620689653,
      "grad_norm": 4.5200910568237305,
      "learning_rate": 4.920557351506409e-06,
      "logits/chosen": -2.358900547027588,
      "logits/rejected": -2.3591980934143066,
      "logps/chosen": -82.83473205566406,
      "logps/rejected": -80.66204071044922,
      "loss": 0.5996,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.046993009746074677,
      "rewards/margins": 0.20401433110237122,
      "rewards/rejected": -0.2510073184967041,
      "step": 100
    },
    {
      "epoch": 3.793103448275862,
      "grad_norm": 4.106562614440918,
      "learning_rate": 4.878569458453592e-06,
      "logits/chosen": -2.347762107849121,
      "logits/rejected": -2.33540415763855,
      "logps/chosen": -72.23506164550781,
      "logps/rejected": -85.77088928222656,
      "loss": 0.6003,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.051100969314575195,
      "rewards/margins": 0.24443332850933075,
      "rewards/rejected": -0.29553431272506714,
      "step": 110
    },
    {
      "epoch": 4.137931034482759,
      "grad_norm": 4.67632532119751,
      "learning_rate": 4.827968782785062e-06,
      "logits/chosen": -2.353370428085327,
      "logits/rejected": -2.378964900970459,
      "logps/chosen": -67.91919708251953,
      "logps/rejected": -86.26390838623047,
      "loss": 0.5716,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.11952020972967148,
      "rewards/margins": 0.30119410157203674,
      "rewards/rejected": -0.42071428894996643,
      "step": 120
    },
    {
      "epoch": 4.482758620689655,
      "grad_norm": 4.5131025314331055,
      "learning_rate": 4.7689385491773934e-06,
      "logits/chosen": -2.3820648193359375,
      "logits/rejected": -2.403965473175049,
      "logps/chosen": -71.66841125488281,
      "logps/rejected": -77.90869140625,
      "loss": 0.5362,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.0694696456193924,
      "rewards/margins": 0.47986412048339844,
      "rewards/rejected": -0.5493337512016296,
      "step": 130
    },
    {
      "epoch": 4.827586206896552,
      "grad_norm": 4.446605205535889,
      "learning_rate": 4.70169250567482e-06,
      "logits/chosen": -2.372788906097412,
      "logits/rejected": -2.379612684249878,
      "logps/chosen": -69.44779968261719,
      "logps/rejected": -80.1120376586914,
      "loss": 0.5186,
      "rewards/accuracies": 0.862500011920929,
      "rewards/chosen": -0.12896183133125305,
      "rewards/margins": 0.5137210488319397,
      "rewards/rejected": -0.6426829099655151,
      "step": 140
    },
    {
      "epoch": 5.172413793103448,
      "grad_norm": 4.605374813079834,
      "learning_rate": 4.626474149709127e-06,
      "logits/chosen": -2.3140041828155518,
      "logits/rejected": -2.336975574493408,
      "logps/chosen": -87.05641174316406,
      "logps/rejected": -94.53668212890625,
      "loss": 0.4794,
      "rewards/accuracies": 0.8374999761581421,
      "rewards/chosen": -0.15513856709003448,
      "rewards/margins": 0.6417296528816223,
      "rewards/rejected": -0.7968682050704956,
      "step": 150
    },
    {
      "epoch": 5.517241379310345,
      "grad_norm": 6.027809143066406,
      "learning_rate": 4.54355584639723e-06,
      "logits/chosen": -2.365192413330078,
      "logits/rejected": -2.3872745037078857,
      "logps/chosen": -73.7781753540039,
      "logps/rejected": -82.1873550415039,
      "loss": 0.4599,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.25834327936172485,
      "rewards/margins": 0.5800828337669373,
      "rewards/rejected": -0.8384261131286621,
      "step": 160
    },
    {
      "epoch": 5.862068965517241,
      "grad_norm": 5.196141242980957,
      "learning_rate": 4.45323784230908e-06,
      "logits/chosen": -2.391653299331665,
      "logits/rejected": -2.394854784011841,
      "logps/chosen": -72.33963775634766,
      "logps/rejected": -86.77699279785156,
      "loss": 0.4467,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.2543284595012665,
      "rewards/margins": 0.7806032299995422,
      "rewards/rejected": -1.0349315404891968,
      "step": 170
    },
    {
      "epoch": 6.206896551724138,
      "grad_norm": 5.285089015960693,
      "learning_rate": 4.355847178277025e-06,
      "logits/chosen": -2.3918795585632324,
      "logits/rejected": -2.4050662517547607,
      "logps/chosen": -77.20832061767578,
      "logps/rejected": -98.5592041015625,
      "loss": 0.4052,
      "rewards/accuracies": 0.887499988079071,
      "rewards/chosen": -0.35020238161087036,
      "rewards/margins": 1.0169063806533813,
      "rewards/rejected": -1.367108702659607,
      "step": 180
    },
    {
      "epoch": 6.551724137931035,
      "grad_norm": 5.308840751647949,
      "learning_rate": 4.2517365051833564e-06,
      "logits/chosen": -2.3778843879699707,
      "logits/rejected": -2.3865675926208496,
      "logps/chosen": -69.90904235839844,
      "logps/rejected": -86.66047668457031,
      "loss": 0.3809,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -0.42647725343704224,
      "rewards/margins": 0.9949015378952026,
      "rewards/rejected": -1.4213788509368896,
      "step": 190
    },
    {
      "epoch": 6.896551724137931,
      "grad_norm": 6.095545768737793,
      "learning_rate": 4.141282807014034e-06,
      "logits/chosen": -2.407437801361084,
      "logits/rejected": -2.4296727180480957,
      "logps/chosen": -82.95701599121094,
      "logps/rejected": -83.4615478515625,
      "loss": 0.3742,
      "rewards/accuracies": 0.875,
      "rewards/chosen": -0.6070786714553833,
      "rewards/margins": 0.968428909778595,
      "rewards/rejected": -1.5755075216293335,
      "step": 200
    },
    {
      "epoch": 7.241379310344827,
      "grad_norm": 5.683840751647949,
      "learning_rate": 4.024886035802432e-06,
      "logits/chosen": -2.411830186843872,
      "logits/rejected": -2.4354729652404785,
      "logps/chosen": -61.66484451293945,
      "logps/rejected": -90.95805358886719,
      "loss": 0.3421,
      "rewards/accuracies": 0.8999999761581421,
      "rewards/chosen": -0.5454065203666687,
      "rewards/margins": 1.2011836767196655,
      "rewards/rejected": -1.7465900182724,
      "step": 210
    },
    {
      "epoch": 7.586206896551724,
      "grad_norm": 5.584890842437744,
      "learning_rate": 3.9029676634059565e-06,
      "logits/chosen": -2.3816287517547607,
      "logits/rejected": -2.37958025932312,
      "logps/chosen": -87.12813568115234,
      "logps/rejected": -92.26463317871094,
      "loss": 0.3138,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.5636588335037231,
      "rewards/margins": 1.3944618701934814,
      "rewards/rejected": -1.9581207036972046,
      "step": 220
    },
    {
      "epoch": 7.931034482758621,
      "grad_norm": 7.526435375213623,
      "learning_rate": 3.7759691553595214e-06,
      "logits/chosen": -2.399634599685669,
      "logits/rejected": -2.407938241958618,
      "logps/chosen": -82.55473327636719,
      "logps/rejected": -89.76951599121094,
      "loss": 0.3046,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -0.9372364282608032,
      "rewards/margins": 1.309622049331665,
      "rewards/rejected": -2.246858596801758,
      "step": 230
    },
    {
      "epoch": 8.275862068965518,
      "grad_norm": 5.150575637817383,
      "learning_rate": 3.6443503723320837e-06,
      "logits/chosen": -2.36824893951416,
      "logits/rejected": -2.3758621215820312,
      "logps/chosen": -85.68852996826172,
      "logps/rejected": -102.09485626220703,
      "loss": 0.2638,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9749407768249512,
      "rewards/margins": 1.6947702169418335,
      "rewards/rejected": -2.669710874557495,
      "step": 240
    },
    {
      "epoch": 8.620689655172415,
      "grad_norm": 6.282366752624512,
      "learning_rate": 3.508587904974522e-06,
      "logits/chosen": -2.3659470081329346,
      "logits/rejected": -2.367462635040283,
      "logps/chosen": -87.00023651123047,
      "logps/rejected": -123.0126953125,
      "loss": 0.2405,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.026289939880371,
      "rewards/margins": 1.9990599155426025,
      "rewards/rejected": -3.0253496170043945,
      "step": 250
    },
    {
      "epoch": 8.96551724137931,
      "grad_norm": 6.396641731262207,
      "learning_rate": 3.3691733481883693e-06,
      "logits/chosen": -2.3716306686401367,
      "logits/rejected": -2.3546042442321777,
      "logps/chosen": -87.29267883300781,
      "logps/rejected": -98.01856994628906,
      "loss": 0.2546,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2485218048095703,
      "rewards/margins": 1.980831503868103,
      "rewards/rejected": -3.229353427886963,
      "step": 260
    },
    {
      "epoch": 9.310344827586206,
      "grad_norm": 5.644590377807617,
      "learning_rate": 3.226611521064278e-06,
      "logits/chosen": -2.2916407585144043,
      "logits/rejected": -2.3004841804504395,
      "logps/chosen": -79.30825805664062,
      "logps/rejected": -113.4272689819336,
      "loss": 0.1999,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -1.1944876909255981,
      "rewards/margins": 2.365144729614258,
      "rewards/rejected": -3.5596320629119873,
      "step": 270
    },
    {
      "epoch": 9.655172413793103,
      "grad_norm": 6.177416801452637,
      "learning_rate": 3.0814186389357765e-06,
      "logits/chosen": -2.346484661102295,
      "logits/rejected": -2.346285581588745,
      "logps/chosen": -77.01927947998047,
      "logps/rejected": -105.2061538696289,
      "loss": 0.1875,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -1.260752558708191,
      "rewards/margins": 2.249293565750122,
      "rewards/rejected": -3.5100464820861816,
      "step": 280
    },
    {
      "epoch": 10.0,
      "grad_norm": 6.281085014343262,
      "learning_rate": 2.9341204441673267e-06,
      "logits/chosen": -2.3005917072296143,
      "logits/rejected": -2.3319716453552246,
      "logps/chosen": -92.97987365722656,
      "logps/rejected": -112.0716552734375,
      "loss": 0.2036,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.6851022243499756,
      "rewards/margins": 2.1508073806762695,
      "rewards/rejected": -3.835909605026245,
      "step": 290
    },
    {
      "epoch": 10.344827586206897,
      "grad_norm": 5.364060401916504,
      "learning_rate": 2.785250302445062e-06,
      "logits/chosen": -2.3305928707122803,
      "logits/rejected": -2.3451201915740967,
      "logps/chosen": -85.11759185791016,
      "logps/rejected": -115.65193176269531,
      "loss": 0.1664,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -1.5494135618209839,
      "rewards/margins": 2.6376235485076904,
      "rewards/rejected": -4.187037467956543,
      "step": 300
    },
    {
      "epoch": 10.689655172413794,
      "grad_norm": 5.540137767791748,
      "learning_rate": 2.6353472714635443e-06,
      "logits/chosen": -2.3217194080352783,
      "logits/rejected": -2.345512628555298,
      "logps/chosen": -83.56015014648438,
      "logps/rejected": -111.0335922241211,
      "loss": 0.1446,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -1.6171716451644897,
      "rewards/margins": 2.553410053253174,
      "rewards/rejected": -4.170581340789795,
      "step": 310
    },
    {
      "epoch": 11.03448275862069,
      "grad_norm": 5.321618556976318,
      "learning_rate": 2.4849541490017868e-06,
      "logits/chosen": -2.2711265087127686,
      "logits/rejected": -2.274831533432007,
      "logps/chosen": -93.46397399902344,
      "logps/rejected": -138.03286743164062,
      "loss": 0.1416,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -2.0557093620300293,
      "rewards/margins": 2.809981346130371,
      "rewards/rejected": -4.8656907081604,
      "step": 320
    },
    {
      "epoch": 11.379310344827585,
      "grad_norm": 6.168019771575928,
      "learning_rate": 2.3346155074564712e-06,
      "logits/chosen": -2.2561862468719482,
      "logits/rejected": -2.260565996170044,
      "logps/chosen": -89.53536224365234,
      "logps/rejected": -119.193603515625,
      "loss": 0.1174,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.2866904735565186,
      "rewards/margins": 2.866985321044922,
      "rewards/rejected": -5.1536760330200195,
      "step": 330
    },
    {
      "epoch": 11.724137931034482,
      "grad_norm": 4.970952033996582,
      "learning_rate": 2.184875721949277e-06,
      "logits/chosen": -2.204287052154541,
      "logits/rejected": -2.2466628551483154,
      "logps/chosen": -171.76095581054688,
      "logps/rejected": -147.26907348632812,
      "loss": 0.1152,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.4709744453430176,
      "rewards/margins": 3.782508134841919,
      "rewards/rejected": -6.253481864929199,
      "step": 340
    },
    {
      "epoch": 12.068965517241379,
      "grad_norm": 4.031501770019531,
      "learning_rate": 2.0362769991485514e-06,
      "logits/chosen": -2.245434522628784,
      "logits/rejected": -2.267470121383667,
      "logps/chosen": -98.18807220458984,
      "logps/rejected": -143.83383178710938,
      "loss": 0.1104,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.5792040824890137,
      "rewards/margins": 3.238723039627075,
      "rewards/rejected": -5.817927360534668,
      "step": 350
    },
    {
      "epoch": 12.413793103448276,
      "grad_norm": 4.009598731994629,
      "learning_rate": 1.8893574139429226e-06,
      "logits/chosen": -2.2339296340942383,
      "logits/rejected": -2.254106283187866,
      "logps/chosen": -104.34129333496094,
      "logps/rejected": -130.74081420898438,
      "loss": 0.0859,
      "rewards/accuracies": 0.9750000238418579,
      "rewards/chosen": -2.8974556922912598,
      "rewards/margins": 3.275568723678589,
      "rewards/rejected": -6.1730241775512695,
      "step": 360
    },
    {
      "epoch": 12.758620689655173,
      "grad_norm": 6.944422721862793,
      "learning_rate": 1.744648961076068e-06,
      "logits/chosen": -2.252500295639038,
      "logits/rejected": -2.25302791595459,
      "logps/chosen": -97.11503601074219,
      "logps/rejected": -135.61911010742188,
      "loss": 0.0901,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.814434051513672,
      "rewards/margins": 3.549976348876953,
      "rewards/rejected": -6.364409923553467,
      "step": 370
    },
    {
      "epoch": 13.10344827586207,
      "grad_norm": 5.268442630767822,
      "learning_rate": 1.602675628797636e-06,
      "logits/chosen": -2.2403993606567383,
      "logits/rejected": -2.240546703338623,
      "logps/chosen": -101.62408447265625,
      "logps/rejected": -136.84268188476562,
      "loss": 0.0902,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -2.9260125160217285,
      "rewards/margins": 3.421342134475708,
      "rewards/rejected": -6.347354888916016,
      "step": 380
    },
    {
      "epoch": 13.448275862068966,
      "grad_norm": 4.1959404945373535,
      "learning_rate": 1.4639515015056205e-06,
      "logits/chosen": -2.2759053707122803,
      "logits/rejected": -2.2657554149627686,
      "logps/chosen": -103.3962631225586,
      "logps/rejected": -138.56057739257812,
      "loss": 0.0729,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.688096523284912,
      "rewards/margins": 4.030136585235596,
      "rewards/rejected": -6.718233585357666,
      "step": 390
    },
    {
      "epoch": 13.793103448275861,
      "grad_norm": 4.1255574226379395,
      "learning_rate": 1.328978898250525e-06,
      "logits/chosen": -2.180371046066284,
      "logits/rejected": -2.213881015777588,
      "logps/chosen": -100.98771667480469,
      "logps/rejected": -135.07044982910156,
      "loss": 0.0747,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.0276432037353516,
      "rewards/margins": 3.8850128650665283,
      "rewards/rejected": -6.912655830383301,
      "step": 400
    },
    {
      "epoch": 14.137931034482758,
      "grad_norm": 5.081278324127197,
      "learning_rate": 1.198246553841744e-06,
      "logits/chosen": -2.237619161605835,
      "logits/rejected": -2.219052314758301,
      "logps/chosen": -98.90690612792969,
      "logps/rejected": -131.66647338867188,
      "loss": 0.0576,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.216820478439331,
      "rewards/margins": 3.5623440742492676,
      "rewards/rejected": -6.779164791107178,
      "step": 410
    },
    {
      "epoch": 14.482758620689655,
      "grad_norm": 4.580889701843262,
      "learning_rate": 1.0722278491423998e-06,
      "logits/chosen": -2.1594581604003906,
      "logits/rejected": -2.183774471282959,
      "logps/chosen": -106.17756652832031,
      "logps/rejected": -163.53775024414062,
      "loss": 0.0545,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.6679279804229736,
      "rewards/margins": 4.358443260192871,
      "rewards/rejected": -8.026371955871582,
      "step": 420
    },
    {
      "epoch": 14.827586206896552,
      "grad_norm": 7.627413749694824,
      "learning_rate": 9.513790969606926e-07,
      "logits/chosen": -2.1865813732147217,
      "logits/rejected": -2.2028133869171143,
      "logps/chosen": -109.03575134277344,
      "logps/rejected": -147.6363983154297,
      "loss": 0.0552,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.6653411388397217,
      "rewards/margins": 3.91802978515625,
      "rewards/rejected": -7.583371162414551,
      "step": 430
    },
    {
      "epoch": 15.172413793103448,
      "grad_norm": 4.102783679962158,
      "learning_rate": 8.361378897445643e-07,
      "logits/chosen": -2.18919038772583,
      "logits/rejected": -2.2041501998901367,
      "logps/chosen": -110.42228698730469,
      "logps/rejected": -157.49429321289062,
      "loss": 0.0596,
      "rewards/accuracies": 0.9624999761581421,
      "rewards/chosen": -3.86995005607605,
      "rewards/margins": 4.187633037567139,
      "rewards/rejected": -8.057581901550293,
      "step": 440
    },
    {
      "epoch": 15.517241379310345,
      "grad_norm": 4.329521656036377,
      "learning_rate": 7.269215150626391e-07,
      "logits/chosen": -2.1742970943450928,
      "logits/rejected": -2.1848294734954834,
      "logps/chosen": -107.87260437011719,
      "logps/rejected": -150.42959594726562,
      "loss": 0.047,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.9351372718811035,
      "rewards/margins": 4.330621242523193,
      "rewards/rejected": -8.26575756072998,
      "step": 450
    },
    {
      "epoch": 15.862068965517242,
      "grad_norm": 4.850955963134766,
      "learning_rate": 6.241254446089942e-07,
      "logits/chosen": -2.2147865295410156,
      "logits/rejected": -2.2335610389709473,
      "logps/chosen": -105.22979736328125,
      "logps/rejected": -149.566162109375,
      "loss": 0.0443,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.2662596702575684,
      "rewards/margins": 4.226635932922363,
      "rewards/rejected": -7.49289608001709,
      "step": 460
    },
    {
      "epoch": 16.20689655172414,
      "grad_norm": 3.0453879833221436,
      "learning_rate": 5.281219022030423e-07,
      "logits/chosen": -2.158506155014038,
      "logits/rejected": -2.1904215812683105,
      "logps/chosen": -119.72633361816406,
      "logps/rejected": -159.1144561767578,
      "loss": 0.0409,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -3.4267520904541016,
      "rewards/margins": 4.899367332458496,
      "rewards/rejected": -8.326119422912598,
      "step": 470
    },
    {
      "epoch": 16.551724137931036,
      "grad_norm": 3.7533819675445557,
      "learning_rate": 4.392585159698087e-07,
      "logits/chosen": -2.152306318283081,
      "logits/rejected": -2.1774165630340576,
      "logps/chosen": -112.7719955444336,
      "logps/rejected": -165.19509887695312,
      "loss": 0.0416,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.251755714416504,
      "rewards/margins": 4.445396900177002,
      "rewards/rejected": -8.697153091430664,
      "step": 480
    },
    {
      "epoch": 16.896551724137932,
      "grad_norm": 7.166266918182373,
      "learning_rate": 3.578570595810274e-07,
      "logits/chosen": -2.1818134784698486,
      "logits/rejected": -2.2025341987609863,
      "logps/chosen": -96.93048095703125,
      "logps/rejected": -171.3131103515625,
      "loss": 0.0417,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.020156383514404,
      "rewards/margins": 5.221116542816162,
      "rewards/rejected": -9.241272926330566,
      "step": 490
    },
    {
      "epoch": 17.24137931034483,
      "grad_norm": 3.491783618927002,
      "learning_rate": 2.8421228711503127e-07,
      "logits/chosen": -2.1831130981445312,
      "logits/rejected": -2.1822221279144287,
      "logps/chosen": -113.05125427246094,
      "logps/rejected": -171.09835815429688,
      "loss": 0.0369,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.272199630737305,
      "rewards/margins": 4.791441440582275,
      "rewards/rejected": -9.063641548156738,
      "step": 500
    },
    {
      "epoch": 17.586206896551722,
      "grad_norm": 2.9002482891082764,
      "learning_rate": 2.1859086575439225e-07,
      "logits/chosen": -2.211790084838867,
      "logits/rejected": -2.1946873664855957,
      "logps/chosen": -129.7812042236328,
      "logps/rejected": -168.2603302001953,
      "loss": 0.0359,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.544430732727051,
      "rewards/margins": 4.710877418518066,
      "rewards/rejected": -9.2553071975708,
      "step": 510
    },
    {
      "epoch": 17.93103448275862,
      "grad_norm": 3.7184576988220215,
      "learning_rate": 1.6123041018599766e-07,
      "logits/chosen": -2.1540164947509766,
      "logits/rejected": -2.147338390350342,
      "logps/chosen": -117.89579772949219,
      "logps/rejected": -158.124755859375,
      "loss": 0.0413,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.149048328399658,
      "rewards/margins": 4.793323516845703,
      "rewards/rejected": -8.94237232208252,
      "step": 520
    },
    {
      "epoch": 18.275862068965516,
      "grad_norm": 3.723693370819092,
      "learning_rate": 1.1233862220001168e-07,
      "logits/chosen": -2.169210195541382,
      "logits/rejected": -2.184584140777588,
      "logps/chosen": -107.45458984375,
      "logps/rejected": -161.4453582763672,
      "loss": 0.0356,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -3.943101167678833,
      "rewards/margins": 5.099206447601318,
      "rewards/rejected": -9.04230785369873,
      "step": 530
    },
    {
      "epoch": 18.620689655172413,
      "grad_norm": 2.663479804992676,
      "learning_rate": 7.209253860320897e-08,
      "logits/chosen": -2.141265392303467,
      "logits/rejected": -2.1849865913391113,
      "logps/chosen": -155.58615112304688,
      "logps/rejected": -161.23976135253906,
      "loss": 0.0346,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -2.961081027984619,
      "rewards/margins": 6.033589839935303,
      "rewards/rejected": -8.994670867919922,
      "step": 540
    },
    {
      "epoch": 18.96551724137931,
      "grad_norm": 3.3965961933135986,
      "learning_rate": 4.063789016999331e-08,
      "logits/chosen": -2.2001423835754395,
      "logits/rejected": -2.214458703994751,
      "logps/chosen": -128.25711059570312,
      "logps/rejected": -166.47296142578125,
      "loss": 0.0394,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.388108730316162,
      "rewards/margins": 4.831867694854736,
      "rewards/rejected": -9.219977378845215,
      "step": 550
    },
    {
      "epoch": 19.310344827586206,
      "grad_norm": 3.214250326156616,
      "learning_rate": 1.808857395232788e-08,
      "logits/chosen": -2.147885799407959,
      "logits/rejected": -2.1538643836975098,
      "logps/chosen": -119.9779281616211,
      "logps/rejected": -163.17922973632812,
      "loss": 0.0352,
      "rewards/accuracies": 0.987500011920929,
      "rewards/chosen": -4.499241828918457,
      "rewards/margins": 4.586289882659912,
      "rewards/rejected": -9.085531234741211,
      "step": 560
    },
    {
      "epoch": 19.655172413793103,
      "grad_norm": 3.0123510360717773,
      "learning_rate": 4.526240859345499e-09,
      "logits/chosen": -2.1862876415252686,
      "logits/rejected": -2.1756527423858643,
      "logps/chosen": -135.414306640625,
      "logps/rejected": -169.79623413085938,
      "loss": 0.0344,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.599732398986816,
      "rewards/margins": 4.528085231781006,
      "rewards/rejected": -9.12781810760498,
      "step": 570
    },
    {
      "epoch": 20.0,
      "grad_norm": 2.647522449493408,
      "learning_rate": 0.0,
      "logits/chosen": -2.175386428833008,
      "logits/rejected": -2.183178424835205,
      "logps/chosen": -106.87957763671875,
      "logps/rejected": -163.1853790283203,
      "loss": 0.036,
      "rewards/accuracies": 1.0,
      "rewards/chosen": -4.439215183258057,
      "rewards/margins": 4.813824653625488,
      "rewards/rejected": -9.253039360046387,
      "step": 580
    },
    {
      "epoch": 20.0,
      "step": 580,
      "total_flos": 2.1130760508114207e+18,
      "train_loss": 0.27193391800954425,
      "train_runtime": 4447.2543,
      "train_samples_per_second": 8.324,
      "train_steps_per_second": 0.13
    }
  ],
  "logging_steps": 10,
  "max_steps": 580,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 20,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 2.1130760508114207e+18,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}