{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.940346752865119,
  "eval_steps": 35,
  "global_step": 600,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.023508668821627974,
      "grad_norm": 94.17500305175781,
      "learning_rate": 9.333333333333334e-08,
      "logits/chosen": -2.9187049865722656,
      "logits/rejected": -2.75704288482666,
      "logps/chosen": -469.19525146484375,
      "logps/rejected": -380.0172424316406,
      "loss": 0.6763,
      "rewards/accuracies": 0.5375000238418579,
      "rewards/chosen": 0.08780575543642044,
      "rewards/margins": 0.040122274309396744,
      "rewards/rejected": 0.047683484852313995,
      "step": 15
    },
    {
      "epoch": 0.04701733764325595,
      "grad_norm": 91.0787353515625,
      "learning_rate": 1.9333333333333332e-07,
      "logits/chosen": -2.999070644378662,
      "logits/rejected": -2.760476589202881,
      "logps/chosen": -468.59783935546875,
      "logps/rejected": -361.46875,
      "loss": 0.6239,
      "rewards/accuracies": 0.6875,
      "rewards/chosen": 0.42466142773628235,
      "rewards/margins": 0.18366314470767975,
      "rewards/rejected": 0.2409982681274414,
      "step": 30
    },
    {
      "epoch": 0.05485356058379861,
      "eval_logits/chosen": -2.9845082759857178,
      "eval_logits/rejected": -2.7968697547912598,
      "eval_logps/chosen": -455.1330261230469,
      "eval_logps/rejected": -369.554931640625,
      "eval_loss": 0.612461507320404,
      "eval_rewards/accuracies": 0.6765195727348328,
      "eval_rewards/chosen": 0.592057466506958,
      "eval_rewards/margins": 0.23903077840805054,
      "eval_rewards/rejected": 0.35302671790122986,
      "eval_runtime": 181.9833,
      "eval_samples_per_second": 13.199,
      "eval_steps_per_second": 6.6,
      "step": 35
    },
    {
      "epoch": 0.07052600646488393,
      "grad_norm": 79.9468994140625,
      "learning_rate": 2.933333333333333e-07,
      "logits/chosen": -3.013662099838257,
      "logits/rejected": -2.8054909706115723,
      "logps/chosen": -446.2762756347656,
      "logps/rejected": -384.8152770996094,
      "loss": 0.6178,
      "rewards/accuracies": 0.6645833253860474,
      "rewards/chosen": 0.615195631980896,
      "rewards/margins": 0.2345016449689865,
      "rewards/rejected": 0.3806939721107483,
      "step": 45
    },
    {
      "epoch": 0.0940346752865119,
      "grad_norm": 77.7275619506836,
      "learning_rate": 3.933333333333333e-07,
      "logits/chosen": -2.934065341949463,
      "logits/rejected": -2.7790465354919434,
      "logps/chosen": -489.3516540527344,
      "logps/rejected": -384.1716613769531,
      "loss": 0.5679,
      "rewards/accuracies": 0.7166666388511658,
      "rewards/chosen": 0.8837584853172302,
      "rewards/margins": 0.41531360149383545,
      "rewards/rejected": 0.46844482421875,
      "step": 60
    },
    {
      "epoch": 0.10970712116759722,
      "eval_logits/chosen": -2.9480597972869873,
      "eval_logits/rejected": -2.767725706100464,
      "eval_logps/chosen": -450.7250671386719,
      "eval_logps/rejected": -367.63983154296875,
      "eval_loss": 0.5620076656341553,
      "eval_rewards/accuracies": 0.7189841866493225,
      "eval_rewards/chosen": 1.0328545570373535,
      "eval_rewards/margins": 0.4883180558681488,
      "eval_rewards/rejected": 0.5445364713668823,
      "eval_runtime": 182.4653,
      "eval_samples_per_second": 13.164,
      "eval_steps_per_second": 6.582,
      "step": 70
    },
    {
      "epoch": 0.11754334410813988,
      "grad_norm": 72.56718444824219,
      "learning_rate": 4.933333333333333e-07,
      "logits/chosen": -2.913278818130493,
      "logits/rejected": -2.7157583236694336,
      "logps/chosen": -438.0467529296875,
      "logps/rejected": -365.61798095703125,
      "loss": 0.5657,
      "rewards/accuracies": 0.7145833373069763,
      "rewards/chosen": 0.9045347571372986,
      "rewards/margins": 0.449837327003479,
      "rewards/rejected": 0.4546974301338196,
      "step": 75
    },
    {
      "epoch": 0.14105201292976785,
      "grad_norm": 79.41279602050781,
      "learning_rate": 5.933333333333334e-07,
      "logits/chosen": -2.9613964557647705,
      "logits/rejected": -2.75221586227417,
      "logps/chosen": -455.7210388183594,
      "logps/rejected": -348.0450744628906,
      "loss": 0.5183,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 1.0422641038894653,
      "rewards/margins": 0.6262082457542419,
      "rewards/rejected": 0.4160558581352234,
      "step": 90
    },
    {
      "epoch": 0.16456068175139582,
      "grad_norm": 71.57633209228516,
      "learning_rate": 6.933333333333333e-07,
      "logits/chosen": -2.938500165939331,
      "logits/rejected": -2.73604416847229,
      "logps/chosen": -463.0316467285156,
      "logps/rejected": -380.5111389160156,
      "loss": 0.5407,
      "rewards/accuracies": 0.7229166626930237,
      "rewards/chosen": 1.022507905960083,
      "rewards/margins": 0.6094505190849304,
      "rewards/rejected": 0.41305726766586304,
      "step": 105
    },
    {
      "epoch": 0.16456068175139582,
      "eval_logits/chosen": -2.985232353210449,
      "eval_logits/rejected": -2.8150649070739746,
      "eval_logps/chosen": -450.6111145019531,
      "eval_logps/rejected": -369.0615234375,
      "eval_loss": 0.5306517481803894,
      "eval_rewards/accuracies": 0.741881787776947,
      "eval_rewards/chosen": 1.0442498922348022,
      "eval_rewards/margins": 0.641879141330719,
      "eval_rewards/rejected": 0.40237072110176086,
      "eval_runtime": 182.4384,
      "eval_samples_per_second": 13.166,
      "eval_steps_per_second": 6.583,
      "step": 105
    },
    {
      "epoch": 0.1880693505730238,
      "grad_norm": 69.94996643066406,
      "learning_rate": 7.933333333333333e-07,
      "logits/chosen": -2.9730515480041504,
      "logits/rejected": -2.772322654724121,
      "logps/chosen": -468.67315673828125,
      "logps/rejected": -395.2209167480469,
      "loss": 0.5009,
      "rewards/accuracies": 0.7666666507720947,
      "rewards/chosen": 1.1156314611434937,
      "rewards/margins": 0.7098201513290405,
      "rewards/rejected": 0.4058113694190979,
      "step": 120
    },
    {
      "epoch": 0.2115780193946518,
      "grad_norm": 77.51742553710938,
      "learning_rate": 8.933333333333333e-07,
      "logits/chosen": -2.9735770225524902,
      "logits/rejected": -2.7964773178100586,
      "logps/chosen": -449.1340637207031,
      "logps/rejected": -364.1695251464844,
      "loss": 0.5277,
      "rewards/accuracies": 0.7541666626930237,
      "rewards/chosen": 1.1800711154937744,
      "rewards/margins": 0.6686728000640869,
      "rewards/rejected": 0.5113983750343323,
      "step": 135
    },
    {
      "epoch": 0.21941424233519444,
      "eval_logits/chosen": -3.01723575592041,
      "eval_logits/rejected": -2.856294631958008,
      "eval_logps/chosen": -449.0035400390625,
      "eval_logps/rejected": -368.5563659667969,
      "eval_loss": 0.5107097625732422,
      "eval_rewards/accuracies": 0.7635303735733032,
      "eval_rewards/chosen": 1.2050074338912964,
      "eval_rewards/margins": 0.7521231174468994,
      "eval_rewards/rejected": 0.4528845250606537,
      "eval_runtime": 182.658,
      "eval_samples_per_second": 13.15,
      "eval_steps_per_second": 6.575,
      "step": 140
    },
    {
      "epoch": 0.23508668821627976,
      "grad_norm": 86.60038757324219,
      "learning_rate": 9.933333333333333e-07,
      "logits/chosen": -2.9938228130340576,
      "logits/rejected": -2.764103651046753,
      "logps/chosen": -459.8388671875,
      "logps/rejected": -371.2470703125,
      "loss": 0.5268,
      "rewards/accuracies": 0.7520833611488342,
      "rewards/chosen": 1.2029531002044678,
      "rewards/margins": 0.7044091820716858,
      "rewards/rejected": 0.4985438585281372,
      "step": 150
    },
    {
      "epoch": 0.2585953570379077,
      "grad_norm": 62.65684509277344,
      "learning_rate": 9.998451182298019e-07,
      "logits/chosen": -2.98136305809021,
      "logits/rejected": -2.761441707611084,
      "logps/chosen": -436.3692321777344,
      "logps/rejected": -367.0105285644531,
      "loss": 0.493,
      "rewards/accuracies": 0.7729166746139526,
      "rewards/chosen": 1.189981460571289,
      "rewards/margins": 0.7949865460395813,
      "rewards/rejected": 0.39499494433403015,
      "step": 165
    },
    {
      "epoch": 0.27426780291899305,
      "eval_logits/chosen": -3.0445635318756104,
      "eval_logits/rejected": -2.888272523880005,
      "eval_logps/chosen": -449.422607421875,
      "eval_logps/rejected": -369.7976379394531,
      "eval_loss": 0.4949776828289032,
      "eval_rewards/accuracies": 0.7785179018974304,
      "eval_rewards/chosen": 1.1631028652191162,
      "eval_rewards/margins": 0.8343473076820374,
      "eval_rewards/rejected": 0.32875561714172363,
      "eval_runtime": 182.4446,
      "eval_samples_per_second": 13.166,
      "eval_steps_per_second": 6.583,
      "step": 175
    },
    {
      "epoch": 0.2821040258595357,
      "grad_norm": 65.13655090332031,
      "learning_rate": 9.993355436786068e-07,
      "logits/chosen": -3.0274109840393066,
      "logits/rejected": -2.807159185409546,
      "logps/chosen": -451.6954345703125,
      "logps/rejected": -389.3435974121094,
      "loss": 0.5027,
      "rewards/accuracies": 0.7458333373069763,
      "rewards/chosen": 1.1719075441360474,
      "rewards/margins": 0.8119107484817505,
      "rewards/rejected": 0.3599967360496521,
      "step": 180
    },
    {
      "epoch": 0.3056126946811637,
      "grad_norm": 63.794944763183594,
      "learning_rate": 9.98470848495185e-07,
      "logits/chosen": -3.018803358078003,
      "logits/rejected": -2.8374457359313965,
      "logps/chosen": -467.7348937988281,
      "logps/rejected": -358.5385437011719,
      "loss": 0.4892,
      "rewards/accuracies": 0.7729166746139526,
      "rewards/chosen": 1.3426380157470703,
      "rewards/margins": 0.8895413279533386,
      "rewards/rejected": 0.45309677720069885,
      "step": 195
    },
    {
      "epoch": 0.32912136350279164,
      "grad_norm": 59.325931549072266,
      "learning_rate": 9.972516476389642e-07,
      "logits/chosen": -3.010078191757202,
      "logits/rejected": -2.8023157119750977,
      "logps/chosen": -483.93109130859375,
      "logps/rejected": -363.4820861816406,
      "loss": 0.4863,
      "rewards/accuracies": 0.7895833253860474,
      "rewards/chosen": 1.499975323677063,
      "rewards/margins": 0.9867887496948242,
      "rewards/rejected": 0.5131866931915283,
      "step": 210
    },
    {
      "epoch": 0.32912136350279164,
      "eval_logits/chosen": -2.9908792972564697,
      "eval_logits/rejected": -2.841421127319336,
      "eval_logps/chosen": -445.6985778808594,
      "eval_logps/rejected": -367.1920166015625,
      "eval_loss": 0.4917643070220947,
      "eval_rewards/accuracies": 0.7714404463768005,
      "eval_rewards/chosen": 1.5355042219161987,
      "eval_rewards/margins": 0.9461879134178162,
      "eval_rewards/rejected": 0.5893163084983826,
      "eval_runtime": 182.823,
      "eval_samples_per_second": 13.138,
      "eval_steps_per_second": 6.569,
      "step": 210
    },
    {
      "epoch": 0.35263003232441964,
      "grad_norm": 70.83655548095703,
      "learning_rate": 9.956788081889744e-07,
      "logits/chosen": -2.960338592529297,
      "logits/rejected": -2.795625925064087,
      "logps/chosen": -420.8348693847656,
      "logps/rejected": -371.9022216796875,
      "loss": 0.5069,
      "rewards/accuracies": 0.7541666626930237,
      "rewards/chosen": 1.327363133430481,
      "rewards/margins": 0.8786435723304749,
      "rewards/rejected": 0.4487195312976837,
      "step": 225
    },
    {
      "epoch": 0.3761387011460476,
      "grad_norm": 68.10982513427734,
      "learning_rate": 9.93753448727193e-07,
      "logits/chosen": -3.019317865371704,
      "logits/rejected": -2.824141502380371,
      "logps/chosen": -454.7313232421875,
      "logps/rejected": -362.6961669921875,
      "loss": 0.4525,
      "rewards/accuracies": 0.800000011920929,
      "rewards/chosen": 1.4697461128234863,
      "rewards/margins": 1.0406087636947632,
      "rewards/rejected": 0.4291374385356903,
      "step": 240
    },
    {
      "epoch": 0.3839749240865903,
      "eval_logits/chosen": -2.9617090225219727,
      "eval_logits/rejected": -2.813918113708496,
      "eval_logps/chosen": -444.755615234375,
      "eval_logps/rejected": -366.7734069824219,
      "eval_loss": 0.4914787709712982,
      "eval_rewards/accuracies": 0.7631140947341919,
      "eval_rewards/chosen": 1.6298017501831055,
      "eval_rewards/margins": 0.9986252188682556,
      "eval_rewards/rejected": 0.6311766505241394,
      "eval_runtime": 183.7769,
      "eval_samples_per_second": 13.07,
      "eval_steps_per_second": 6.535,
      "step": 245
    },
    {
      "epoch": 0.3996473699676756,
      "grad_norm": 57.24860382080078,
      "learning_rate": 9.914769385430247e-07,
      "logits/chosen": -2.9585585594177246,
      "logits/rejected": -2.747401475906372,
      "logps/chosen": -453.3490905761719,
      "logps/rejected": -383.33203125,
      "loss": 0.5226,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": 1.5862523317337036,
      "rewards/margins": 0.9523304104804993,
      "rewards/rejected": 0.6339219808578491,
      "step": 255
    },
    {
      "epoch": 0.4231560387893036,
      "grad_norm": 102.9881820678711,
      "learning_rate": 9.888508966594814e-07,
      "logits/chosen": -2.9838056564331055,
      "logits/rejected": -2.80302357673645,
      "logps/chosen": -458.0950012207031,
      "logps/rejected": -371.9687805175781,
      "loss": 0.488,
      "rewards/accuracies": 0.7729166746139526,
      "rewards/chosen": 1.4627233743667603,
      "rewards/margins": 0.95100337266922,
      "rewards/rejected": 0.5117199420928955,
      "step": 270
    },
    {
      "epoch": 0.4388284846703889,
      "eval_logits/chosen": -2.9294016361236572,
      "eval_logits/rejected": -2.7830867767333984,
      "eval_logps/chosen": -445.0923767089844,
      "eval_logps/rejected": -367.69287109375,
      "eval_loss": 0.48294496536254883,
      "eval_rewards/accuracies": 0.7651956677436829,
      "eval_rewards/chosen": 1.5961238145828247,
      "eval_rewards/margins": 1.0568901300430298,
      "eval_rewards/rejected": 0.5392336845397949,
      "eval_runtime": 183.0491,
      "eval_samples_per_second": 13.122,
      "eval_steps_per_second": 6.561,
      "step": 280
    },
    {
      "epoch": 0.4466647076109315,
      "grad_norm": 55.373291015625,
      "learning_rate": 9.85877190681755e-07,
      "logits/chosen": -2.9584264755249023,
      "logits/rejected": -2.8002991676330566,
      "logps/chosen": -467.3027648925781,
      "logps/rejected": -373.3812561035156,
      "loss": 0.4784,
      "rewards/accuracies": 0.7791666388511658,
      "rewards/chosen": 1.5210446119308472,
      "rewards/margins": 1.0039273500442505,
      "rewards/rejected": 0.5171175003051758,
      "step": 285
    },
    {
      "epoch": 0.4701733764325595,
      "grad_norm": 80.90605163574219,
      "learning_rate": 9.825579354690029e-07,
      "logits/chosen": -2.9166040420532227,
      "logits/rejected": -2.7068498134613037,
      "logps/chosen": -443.4498291015625,
      "logps/rejected": -359.88787841796875,
      "loss": 0.4947,
      "rewards/accuracies": 0.7395833134651184,
      "rewards/chosen": 1.549621343612671,
      "rewards/margins": 0.9639669060707092,
      "rewards/rejected": 0.5856544375419617,
      "step": 300
    },
    {
      "epoch": 0.49368204525418746,
      "grad_norm": 58.752540588378906,
      "learning_rate": 9.788954916302867e-07,
      "logits/chosen": -2.918592929840088,
      "logits/rejected": -2.7391176223754883,
      "logps/chosen": -445.65692138671875,
      "logps/rejected": -375.7033386230469,
      "loss": 0.4844,
      "rewards/accuracies": 0.7458333373069763,
      "rewards/chosen": 1.6397162675857544,
      "rewards/margins": 1.1213122606277466,
      "rewards/rejected": 0.5184040069580078,
      "step": 315
    },
    {
      "epoch": 0.49368204525418746,
      "eval_logits/chosen": -2.946758270263672,
      "eval_logits/rejected": -2.8033602237701416,
      "eval_logps/chosen": -445.91162109375,
      "eval_logps/rejected": -368.79913330078125,
      "eval_loss": 0.4768896698951721,
      "eval_rewards/accuracies": 0.7756036520004272,
      "eval_rewards/chosen": 1.5141996145248413,
      "eval_rewards/margins": 1.0855913162231445,
      "eval_rewards/rejected": 0.4286082684993744,
      "eval_runtime": 182.7832,
      "eval_samples_per_second": 13.141,
      "eval_steps_per_second": 6.571,
      "step": 315
    },
    {
      "epoch": 0.5171907140758154,
      "grad_norm": 72.6146011352539,
      "learning_rate": 9.748924638457425e-07,
      "logits/chosen": -3.0164575576782227,
      "logits/rejected": -2.7821667194366455,
      "logps/chosen": -461.7873840332031,
      "logps/rejected": -387.76312255859375,
      "loss": 0.448,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": 1.5970878601074219,
      "rewards/margins": 1.1854947805404663,
      "rewards/rejected": 0.41159313917160034,
      "step": 330
    },
    {
      "epoch": 0.5406993828974435,
      "grad_norm": 78.9859848022461,
      "learning_rate": 9.705516990141652e-07,
      "logits/chosen": -2.9356918334960938,
      "logits/rejected": -2.7638401985168457,
      "logps/chosen": -471.6175231933594,
      "logps/rejected": -419.6571350097656,
      "loss": 0.4583,
      "rewards/accuracies": 0.7583333253860474,
      "rewards/chosen": 1.4139249324798584,
      "rewards/margins": 1.0874642133712769,
      "rewards/rejected": 0.32646065950393677,
      "step": 345
    },
    {
      "epoch": 0.5485356058379861,
      "eval_logits/chosen": -3.0017576217651367,
      "eval_logits/rejected": -2.8627164363861084,
      "eval_logps/chosen": -447.2599792480469,
      "eval_logps/rejected": -370.40399169921875,
      "eval_loss": 0.47239452600479126,
      "eval_rewards/accuracies": 0.7801831960678101,
      "eval_rewards/chosen": 1.3793617486953735,
      "eval_rewards/margins": 1.111243486404419,
      "eval_rewards/rejected": 0.2681184411048889,
      "eval_runtime": 183.3777,
      "eval_samples_per_second": 13.099,
      "eval_steps_per_second": 6.549,
      "step": 350
    },
    {
      "epoch": 0.5642080517190714,
      "grad_norm": 84.21063995361328,
      "learning_rate": 9.658762842283341e-07,
      "logits/chosen": -2.990657329559326,
      "logits/rejected": -2.8259623050689697,
      "logps/chosen": -481.35552978515625,
      "logps/rejected": -396.5709533691406,
      "loss": 0.4491,
      "rewards/accuracies": 0.7895833253860474,
      "rewards/chosen": 1.3929458856582642,
      "rewards/margins": 1.1319301128387451,
      "rewards/rejected": 0.2610156834125519,
      "step": 360
    },
    {
      "epoch": 0.5877167205406993,
      "grad_norm": 69.14201354980469,
      "learning_rate": 9.608695445795146e-07,
      "logits/chosen": -3.019925594329834,
      "logits/rejected": -2.8734469413757324,
      "logps/chosen": -471.0541076660156,
      "logps/rejected": -388.4405517578125,
      "loss": 0.4159,
      "rewards/accuracies": 0.8104166388511658,
      "rewards/chosen": 1.6452898979187012,
      "rewards/margins": 1.3720909357070923,
      "rewards/rejected": 0.27319908142089844,
      "step": 375
    },
    {
      "epoch": 0.6033891664217848,
      "eval_logits/chosen": -2.9827041625976562,
      "eval_logits/rejected": -2.843700408935547,
      "eval_logps/chosen": -445.8433532714844,
      "eval_logps/rejected": -369.46636962890625,
      "eval_loss": 0.47262606024742126,
      "eval_rewards/accuracies": 0.779766857624054,
      "eval_rewards/chosen": 1.5210249423980713,
      "eval_rewards/margins": 1.1591413021087646,
      "eval_rewards/rejected": 0.36188361048698425,
      "eval_runtime": 182.9492,
      "eval_samples_per_second": 13.129,
      "eval_steps_per_second": 6.565,
      "step": 385
    },
    {
      "epoch": 0.6112253893623274,
      "grad_norm": 89.24090576171875,
      "learning_rate": 9.555350407926977e-07,
      "logits/chosen": -2.9598388671875,
      "logits/rejected": -2.8019778728485107,
      "logps/chosen": -458.2909240722656,
      "logps/rejected": -379.5040283203125,
      "loss": 0.5023,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.4271316528320312,
      "rewards/margins": 1.0898312330245972,
      "rewards/rejected": 0.33730027079582214,
      "step": 390
    },
    {
      "epoch": 0.6347340581839553,
      "grad_norm": 84.49832916259766,
      "learning_rate": 9.498765666942621e-07,
      "logits/chosen": -3.053419351577759,
      "logits/rejected": -2.8401293754577637,
      "logps/chosen": -430.2739562988281,
      "logps/rejected": -377.2356872558594,
      "loss": 0.4732,
      "rewards/accuracies": 0.7729166746139526,
      "rewards/chosen": 1.3829543590545654,
      "rewards/margins": 1.1306571960449219,
      "rewards/rejected": 0.25229716300964355,
      "step": 405
    },
    {
      "epoch": 0.6582427270055833,
      "grad_norm": 80.26911926269531,
      "learning_rate": 9.438981465138564e-07,
      "logits/chosen": -2.9762353897094727,
      "logits/rejected": -2.8631834983825684,
      "logps/chosen": -464.1763916015625,
      "logps/rejected": -372.65423583984375,
      "loss": 0.4611,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": 1.389533281326294,
      "rewards/margins": 1.1575042009353638,
      "rewards/rejected": 0.23202911019325256,
      "step": 420
    },
    {
      "epoch": 0.6582427270055833,
      "eval_logits/chosen": -2.979790687561035,
      "eval_logits/rejected": -2.84635329246521,
      "eval_logps/chosen": -445.2568054199219,
      "eval_logps/rejected": -369.1412048339844,
      "eval_loss": 0.46475231647491455,
      "eval_rewards/accuracies": 0.779766857624054,
      "eval_rewards/chosen": 1.5796784162521362,
      "eval_rewards/margins": 1.1852765083312988,
      "eval_rewards/rejected": 0.3944019079208374,
      "eval_runtime": 183.2941,
      "eval_samples_per_second": 13.105,
      "eval_steps_per_second": 6.552,
      "step": 420
    },
    {
      "epoch": 0.6817513958272113,
      "grad_norm": 78.24493408203125,
      "learning_rate": 9.376040320224207e-07,
      "logits/chosen": -2.9707512855529785,
      "logits/rejected": -2.756378412246704,
      "logps/chosen": -444.6748046875,
      "logps/rejected": -364.9356994628906,
      "loss": 0.5195,
      "rewards/accuracies": 0.7416666746139526,
      "rewards/chosen": 1.4916802644729614,
      "rewards/margins": 1.0260388851165771,
      "rewards/rejected": 0.4656413197517395,
      "step": 435
    },
    {
      "epoch": 0.7052600646488393,
      "grad_norm": 65.54131317138672,
      "learning_rate": 9.309986995083865e-07,
      "logits/chosen": -2.9713919162750244,
      "logits/rejected": -2.853391170501709,
      "logps/chosen": -437.1709289550781,
      "logps/rejected": -380.7355041503906,
      "loss": 0.4612,
      "rewards/accuracies": 0.768750011920929,
      "rewards/chosen": 1.518881916999817,
      "rewards/margins": 1.1546814441680908,
      "rewards/rejected": 0.3642004728317261,
      "step": 450
    },
    {
      "epoch": 0.7130962875893819,
      "eval_logits/chosen": -2.9684040546417236,
      "eval_logits/rejected": -2.835289716720581,
      "eval_logps/chosen": -444.91162109375,
      "eval_logps/rejected": -369.0731506347656,
      "eval_loss": 0.4633679687976837,
      "eval_rewards/accuracies": 0.7814321517944336,
      "eval_rewards/chosen": 1.6142022609710693,
      "eval_rewards/margins": 1.2129993438720703,
      "eval_rewards/rejected": 0.4012027978897095,
      "eval_runtime": 183.2161,
      "eval_samples_per_second": 13.11,
      "eval_steps_per_second": 6.555,
      "step": 455
    },
    {
      "epoch": 0.7287687334704672,
      "grad_norm": 49.11262130737305,
      "learning_rate": 9.240868465942004e-07,
      "logits/chosen": -2.9776499271392822,
      "logits/rejected": -2.8117339611053467,
      "logps/chosen": -463.0809020996094,
      "logps/rejected": -372.5666809082031,
      "loss": 0.4164,
      "rewards/accuracies": 0.8062499761581421,
      "rewards/chosen": 1.703311800956726,
      "rewards/margins": 1.3144255876541138,
      "rewards/rejected": 0.3888862729072571,
      "step": 465
    },
    {
      "epoch": 0.7522774022920952,
      "grad_norm": 88.16354370117188,
      "learning_rate": 9.168733888954398e-07,
      "logits/chosen": -2.9618024826049805,
      "logits/rejected": -2.816535472869873,
      "logps/chosen": -462.2527770996094,
      "logps/rejected": -384.9928283691406,
      "loss": 0.4777,
      "rewards/accuracies": 0.7916666865348816,
      "rewards/chosen": 1.523806095123291,
      "rewards/margins": 1.139542818069458,
      "rewards/rejected": 0.3842634856700897,
      "step": 480
    },
    {
      "epoch": 0.7679498481731806,
      "eval_logits/chosen": -3.0340590476989746,
      "eval_logits/rejected": -2.9054691791534424,
      "eval_logps/chosen": -446.62152099609375,
      "eval_logps/rejected": -370.80877685546875,
      "eval_loss": 0.45889604091644287,
      "eval_rewards/accuracies": 0.7880932688713074,
      "eval_rewards/chosen": 1.4432086944580078,
      "eval_rewards/margins": 1.2155660390853882,
      "eval_rewards/rejected": 0.22764264047145844,
      "eval_runtime": 183.4706,
      "eval_samples_per_second": 13.092,
      "eval_steps_per_second": 6.546,
      "step": 490
    },
    {
      "epoch": 0.7757860711137232,
      "grad_norm": 64.71389770507812,
      "learning_rate": 9.093634565248934e-07,
      "logits/chosen": -2.998199462890625,
      "logits/rejected": -2.8670761585235596,
      "logps/chosen": -439.2498474121094,
      "logps/rejected": -379.03826904296875,
      "loss": 0.4378,
      "rewards/accuracies": 0.7833333611488342,
      "rewards/chosen": 1.474950909614563,
      "rewards/margins": 1.2197434902191162,
      "rewards/rejected": 0.2552074193954468,
      "step": 495
    },
    {
      "epoch": 0.7992947399353512,
      "grad_norm": 81.95059967041016,
      "learning_rate": 9.015623904440955e-07,
      "logits/chosen": -3.033099412918091,
      "logits/rejected": -2.872230052947998,
      "logps/chosen": -453.91607666015625,
      "logps/rejected": -385.92999267578125,
      "loss": 0.4317,
      "rewards/accuracies": 0.793749988079071,
      "rewards/chosen": 1.5632601976394653,
      "rewards/margins": 1.298865556716919,
      "rewards/rejected": 0.26439470052719116,
      "step": 510
    },
    {
      "epoch": 0.8228034087569791,
      "grad_norm": 63.492652893066406,
      "learning_rate": 8.934757386649061e-07,
      "logits/chosen": -3.0300612449645996,
      "logits/rejected": -2.860938787460327,
      "logps/chosen": -418.1768798828125,
      "logps/rejected": -359.6458740234375,
      "loss": 0.448,
      "rewards/accuracies": 0.7854166626930237,
      "rewards/chosen": 1.449339747428894,
      "rewards/margins": 1.289171814918518,
      "rewards/rejected": 0.16016803681850433,
      "step": 525
    },
    {
      "epoch": 0.8228034087569791,
      "eval_logits/chosen": -3.0507404804229736,
      "eval_logits/rejected": -2.929443836212158,
      "eval_logps/chosen": -446.1484680175781,
      "eval_logps/rejected": -371.0081481933594,
      "eval_loss": 0.4595262110233307,
      "eval_rewards/accuracies": 0.7860116362571716,
      "eval_rewards/chosen": 1.4905126094818115,
      "eval_rewards/margins": 1.2828094959259033,
      "eval_rewards/rejected": 0.20770323276519775,
      "eval_runtime": 184.1856,
      "eval_samples_per_second": 13.041,
      "eval_steps_per_second": 6.521,
      "step": 525
    },
    {
      "epoch": 0.8463120775786072,
      "grad_norm": 62.45206832885742,
      "learning_rate": 8.851092523038417e-07,
      "logits/chosen": -3.0484862327575684,
      "logits/rejected": -2.906770944595337,
      "logps/chosen": -458.0007019042969,
      "logps/rejected": -373.7229919433594,
      "loss": 0.4522,
      "rewards/accuracies": 0.7645833492279053,
      "rewards/chosen": 1.579330325126648,
      "rewards/margins": 1.3272625207901,
      "rewards/rejected": 0.2520677447319031,
      "step": 540
    },
    {
      "epoch": 0.8698207464002351,
      "grad_norm": 79.46917724609375,
      "learning_rate": 8.764688814919589e-07,
      "logits/chosen": -2.996983289718628,
      "logits/rejected": -2.865999937057495,
      "logps/chosen": -481.4201965332031,
      "logps/rejected": -376.74200439453125,
      "loss": 0.4349,
      "rewards/accuracies": 0.7895833253860474,
      "rewards/chosen": 1.7141087055206299,
      "rewards/margins": 1.316036581993103,
      "rewards/rejected": 0.3980720341205597,
      "step": 555
    },
    {
      "epoch": 0.8776569693407777,
      "eval_logits/chosen": -2.9852750301361084,
      "eval_logits/rejected": -2.865750312805176,
      "eval_logps/chosen": -443.4287109375,
      "eval_logps/rejected": -368.6723937988281,
      "eval_loss": 0.4570087790489197,
      "eval_rewards/accuracies": 0.7851790189743042,
      "eval_rewards/chosen": 1.762492060661316,
      "eval_rewards/margins": 1.321212649345398,
      "eval_rewards/rejected": 0.44127941131591797,
      "eval_runtime": 183.4018,
      "eval_samples_per_second": 13.097,
      "eval_steps_per_second": 6.548,
      "step": 560
    },
    {
      "epoch": 0.893329415221863,
      "grad_norm": 84.6218032836914,
      "learning_rate": 8.675607711432023e-07,
      "logits/chosen": -2.9534671306610107,
      "logits/rejected": -2.87934947013855,
      "logps/chosen": -439.9498291015625,
      "logps/rejected": -368.0388488769531,
      "loss": 0.4431,
      "rewards/accuracies": 0.7854166626930237,
      "rewards/chosen": 1.7760926485061646,
      "rewards/margins": 1.3532804250717163,
      "rewards/rejected": 0.42281219363212585,
      "step": 570
    },
    {
      "epoch": 0.916838084043491,
      "grad_norm": 99.58641052246094,
      "learning_rate": 8.583912565842256e-07,
      "logits/chosen": -3.0073468685150146,
      "logits/rejected": -2.877568483352661,
      "logps/chosen": -453.67877197265625,
      "logps/rejected": -404.41595458984375,
      "loss": 0.453,
      "rewards/accuracies": 0.78125,
      "rewards/chosen": 1.554824948310852,
      "rewards/margins": 1.3386683464050293,
      "rewards/rejected": 0.21615657210350037,
      "step": 585
    },
    {
      "epoch": 0.9325105299245764,
      "eval_logits/chosen": -3.022214651107788,
      "eval_logits/rejected": -2.903691530227661,
      "eval_logps/chosen": -444.5338134765625,
      "eval_logps/rejected": -369.7436828613281,
      "eval_loss": 0.4549033045768738,
      "eval_rewards/accuracies": 0.7876769304275513,
      "eval_rewards/chosen": 1.651978611946106,
      "eval_rewards/margins": 1.317826271057129,
      "eval_rewards/rejected": 0.3341525197029114,
      "eval_runtime": 183.2329,
      "eval_samples_per_second": 13.109,
      "eval_steps_per_second": 6.555,
      "step": 595
    },
    {
      "epoch": 0.940346752865119,
      "grad_norm": 78.86089324951172,
      "learning_rate": 8.489668590487934e-07,
      "logits/chosen": -3.0001256465911865,
      "logits/rejected": -2.8593249320983887,
      "logps/chosen": -477.72747802734375,
      "logps/rejected": -387.3090515136719,
      "loss": 0.4064,
      "rewards/accuracies": 0.8166666626930237,
      "rewards/chosen": 1.7754851579666138,
      "rewards/margins": 1.4204692840576172,
      "rewards/rejected": 0.3550158143043518,
      "step": 600
    }
  ],
  "logging_steps": 15,
  "max_steps": 1917,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 40,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}