{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 1.9979429914781075,
  "eval_steps": 200,
  "global_step": 5100,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.019590557351356647,
      "grad_norm": 140.09786987304688,
      "learning_rate": 3.2666666666666663e-07,
      "logits/chosen": -0.9822076559066772,
      "logits/rejected": -0.6774621605873108,
      "logps/chosen": -352.589111328125,
      "logps/rejected": -284.4425354003906,
      "loss": 0.6901,
      "rewards/accuracies": 0.47999998927116394,
      "rewards/chosen": 0.0031729438342154026,
      "rewards/margins": 0.008149052038788795,
      "rewards/rejected": -0.004976108204573393,
      "step": 50
    },
    {
      "epoch": 0.03918111470271329,
      "grad_norm": 128.5372314453125,
      "learning_rate": 6.6e-07,
      "logits/chosen": -0.9334858059883118,
      "logits/rejected": -0.6409775018692017,
      "logps/chosen": -366.2322692871094,
      "logps/rejected": -287.4626770019531,
      "loss": 0.6758,
      "rewards/accuracies": 0.6150000095367432,
      "rewards/chosen": 0.02407059632241726,
      "rewards/margins": 0.04141266271471977,
      "rewards/rejected": -0.017342066392302513,
      "step": 100
    },
    {
      "epoch": 0.05877167205406994,
      "grad_norm": 103.95256805419922,
      "learning_rate": 9.933333333333333e-07,
      "logits/chosen": -0.8153188228607178,
      "logits/rejected": -0.5176287293434143,
      "logps/chosen": -373.8503112792969,
      "logps/rejected": -307.275634765625,
      "loss": 0.6225,
      "rewards/accuracies": 0.6775000095367432,
      "rewards/chosen": 0.07690317183732986,
      "rewards/margins": 0.18566985428333282,
      "rewards/rejected": -0.10876669734716415,
      "step": 150
    },
    {
      "epoch": 0.07836222940542659,
      "grad_norm": 130.03802490234375,
      "learning_rate": 9.99894936347371e-07,
      "logits/chosen": -0.8271468877792358,
      "logits/rejected": -0.5104550719261169,
      "logps/chosen": -350.3367614746094,
      "logps/rejected": -296.0983581542969,
      "loss": 0.6152,
      "rewards/accuracies": 0.6775000095367432,
      "rewards/chosen": -0.0007632786291651428,
      "rewards/margins": 0.25347909331321716,
      "rewards/rejected": -0.2542423605918884,
      "step": 200
    },
    {
      "epoch": 0.07836222940542659,
      "eval_logits/chosen": -0.8247441053390503,
      "eval_logits/rejected": -0.5691510438919067,
      "eval_logps/chosen": -352.2530212402344,
      "eval_logps/rejected": -288.40521240234375,
      "eval_loss": 0.6011638641357422,
      "eval_rewards/accuracies": 0.684013307094574,
      "eval_rewards/chosen": 0.006044471170753241,
      "eval_rewards/margins": 0.30330440402030945,
      "eval_rewards/rejected": -0.2972599267959595,
      "eval_runtime": 180.5524,
      "eval_samples_per_second": 13.304,
      "eval_steps_per_second": 6.652,
      "step": 200
    },
    {
      "epoch": 0.09795278675678323,
      "grad_norm": 127.07317352294922,
      "learning_rate": 9.995711712979657e-07,
      "logits/chosen": -0.7803874015808105,
      "logits/rejected": -0.5503089427947998,
      "logps/chosen": -354.3157043457031,
      "logps/rejected": -271.8356018066406,
      "loss": 0.5935,
      "rewards/accuracies": 0.6825000047683716,
      "rewards/chosen": -0.0023533145431429148,
      "rewards/margins": 0.3317827880382538,
      "rewards/rejected": -0.33413612842559814,
      "step": 250
    },
    {
      "epoch": 0.11754334410813988,
      "grad_norm": 107.00403594970703,
      "learning_rate": 9.990288027658056e-07,
      "logits/chosen": -0.8059212565422058,
      "logits/rejected": -0.5893051028251648,
      "logps/chosen": -349.48760986328125,
      "logps/rejected": -285.4340515136719,
      "loss": 0.5528,
      "rewards/accuracies": 0.7599999904632568,
      "rewards/chosen": 0.04797028377652168,
      "rewards/margins": 0.4950464367866516,
      "rewards/rejected": -0.44707614183425903,
      "step": 300
    },
    {
      "epoch": 0.13713390145949653,
      "grad_norm": 118.62214660644531,
      "learning_rate": 9.982680680817391e-07,
      "logits/chosen": -0.9657462239265442,
      "logits/rejected": -0.647233784198761,
      "logps/chosen": -355.55096435546875,
      "logps/rejected": -308.36053466796875,
      "loss": 0.56,
      "rewards/accuracies": 0.7074999809265137,
      "rewards/chosen": -0.11357284337282181,
      "rewards/margins": 0.5093904137611389,
      "rewards/rejected": -0.6229632496833801,
      "step": 350
    },
    {
      "epoch": 0.15672445881085317,
      "grad_norm": 80.19744110107422,
      "learning_rate": 9.972893001297698e-07,
      "logits/chosen": -1.0753679275512695,
      "logits/rejected": -0.731191873550415,
      "logps/chosen": -374.8988952636719,
      "logps/rejected": -319.61199951171875,
      "loss": 0.5683,
      "rewards/accuracies": 0.7049999833106995,
      "rewards/chosen": -0.20595918595790863,
      "rewards/margins": 0.5294285416603088,
      "rewards/rejected": -0.7353877425193787,
      "step": 400
    },
    {
      "epoch": 0.15672445881085317,
      "eval_logits/chosen": -1.0395917892456055,
      "eval_logits/rejected": -0.7957448363304138,
      "eval_logps/chosen": -354.0768127441406,
      "eval_logps/rejected": -293.1625061035156,
      "eval_loss": 0.5456792116165161,
      "eval_rewards/accuracies": 0.7264779210090637,
      "eval_rewards/chosen": -0.17633533477783203,
      "eval_rewards/margins": 0.5966517329216003,
      "eval_rewards/rejected": -0.7729870080947876,
      "eval_runtime": 180.5278,
      "eval_samples_per_second": 13.305,
      "eval_steps_per_second": 6.653,
      "step": 400
    },
    {
      "epoch": 0.17631501616220982,
      "grad_norm": 97.5062026977539,
      "learning_rate": 9.96092927201391e-07,
      "logits/chosen": -1.0175271034240723,
      "logits/rejected": -0.752033531665802,
      "logps/chosen": -355.3705749511719,
      "logps/rejected": -302.32373046875,
      "loss": 0.5547,
      "rewards/accuracies": 0.7024999856948853,
      "rewards/chosen": -0.2221955806016922,
      "rewards/margins": 0.6058707237243652,
      "rewards/rejected": -0.828066349029541,
      "step": 450
    },
    {
      "epoch": 0.19590557351356647,
      "grad_norm": 109.52022552490234,
      "learning_rate": 9.946794728081737e-07,
      "logits/chosen": -1.2173391580581665,
      "logits/rejected": -0.8966682553291321,
      "logps/chosen": -345.02984619140625,
      "logps/rejected": -300.69268798828125,
      "loss": 0.5145,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.26011258363723755,
      "rewards/margins": 0.7154355049133301,
      "rewards/rejected": -0.9755480885505676,
      "step": 500
    },
    {
      "epoch": 0.2154961308649231,
      "grad_norm": 65.85977172851562,
      "learning_rate": 9.930495554526878e-07,
      "logits/chosen": -1.1911191940307617,
      "logits/rejected": -0.9048889875411987,
      "logps/chosen": -360.3658142089844,
      "logps/rejected": -297.0860900878906,
      "loss": 0.5025,
      "rewards/accuracies": 0.7275000214576721,
      "rewards/chosen": -0.28734278678894043,
      "rewards/margins": 0.8380499482154846,
      "rewards/rejected": -1.1253927946090698,
      "step": 550
    },
    {
      "epoch": 0.23508668821627976,
      "grad_norm": 63.237606048583984,
      "learning_rate": 9.912038883578552e-07,
      "logits/chosen": -1.0660635232925415,
      "logits/rejected": -0.7923431992530823,
      "logps/chosen": -363.8102722167969,
      "logps/rejected": -295.3084716796875,
      "loss": 0.5179,
      "rewards/accuracies": 0.7300000190734863,
      "rewards/chosen": -0.4248141050338745,
      "rewards/margins": 0.8246171474456787,
      "rewards/rejected": -1.2494312524795532,
      "step": 600
    },
    {
      "epoch": 0.23508668821627976,
      "eval_logits/chosen": -1.0540363788604736,
      "eval_logits/rejected": -0.8268531560897827,
      "eval_logps/chosen": -355.8159484863281,
      "eval_logps/rejected": -297.4617919921875,
      "eval_loss": 0.5122258067131042,
      "eval_rewards/accuracies": 0.7373022437095642,
      "eval_rewards/chosen": -0.35024747252464294,
      "eval_rewards/margins": 0.8526709675788879,
      "eval_rewards/rejected": -1.2029184103012085,
      "eval_runtime": 181.1919,
      "eval_samples_per_second": 13.257,
      "eval_steps_per_second": 6.628,
      "step": 600
    },
    {
      "epoch": 0.2546772455676364,
      "grad_norm": 194.335693359375,
      "learning_rate": 9.891432791548562e-07,
      "logits/chosen": -1.0396273136138916,
      "logits/rejected": -0.8581287860870361,
      "logps/chosen": -375.4624938964844,
      "logps/rejected": -302.572998046875,
      "loss": 0.4782,
      "rewards/accuracies": 0.7799999713897705,
      "rewards/chosen": -0.3973628580570221,
      "rewards/margins": 0.9747829437255859,
      "rewards/rejected": -1.3721458911895752,
      "step": 650
    },
    {
      "epoch": 0.27426780291899305,
      "grad_norm": 65.69367980957031,
      "learning_rate": 9.868686295297258e-07,
      "logits/chosen": -1.070594072341919,
      "logits/rejected": -0.848624050617218,
      "logps/chosen": -358.2480773925781,
      "logps/rejected": -311.5008850097656,
      "loss": 0.5109,
      "rewards/accuracies": 0.7475000023841858,
      "rewards/chosen": -0.465026319026947,
      "rewards/margins": 0.9383654594421387,
      "rewards/rejected": -1.4033918380737305,
      "step": 700
    },
    {
      "epoch": 0.29385836027034967,
      "grad_norm": 116.7066650390625,
      "learning_rate": 9.8438093482879e-07,
      "logits/chosen": -1.0724475383758545,
      "logits/rejected": -0.8224292993545532,
      "logps/chosen": -373.66754150390625,
      "logps/rejected": -297.4761047363281,
      "loss": 0.483,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -0.39762890338897705,
      "rewards/margins": 1.163187026977539,
      "rewards/rejected": -1.5608159303665161,
      "step": 750
    },
    {
      "epoch": 0.31344891762170635,
      "grad_norm": 124.66069793701172,
      "learning_rate": 9.81681283623121e-07,
      "logits/chosen": -1.0539674758911133,
      "logits/rejected": -0.779601514339447,
      "logps/chosen": -347.1493225097656,
      "logps/rejected": -300.728515625,
      "loss": 0.5427,
      "rewards/accuracies": 0.7250000238418579,
      "rewards/chosen": -0.48545289039611816,
      "rewards/margins": 0.8515650033950806,
      "rewards/rejected": -1.3370177745819092,
      "step": 800
    },
    {
      "epoch": 0.31344891762170635,
      "eval_logits/chosen": -0.9799102544784546,
      "eval_logits/rejected": -0.7651399970054626,
      "eval_logps/chosen": -356.1210021972656,
      "eval_logps/rejected": -299.1585388183594,
      "eval_loss": 0.4938255250453949,
      "eval_rewards/accuracies": 0.7493755221366882,
      "eval_rewards/chosen": -0.38075584173202515,
      "eval_rewards/margins": 0.9918379783630371,
      "eval_rewards/rejected": -1.372593641281128,
      "eval_runtime": 181.2826,
      "eval_samples_per_second": 13.25,
      "eval_steps_per_second": 6.625,
      "step": 800
    },
    {
      "epoch": 0.33303947497306297,
      "grad_norm": 120.1912612915039,
      "learning_rate": 9.787708572321983e-07,
      "logits/chosen": -1.034041404724121,
      "logits/rejected": -0.810564398765564,
      "logps/chosen": -367.707763671875,
      "logps/rejected": -309.762939453125,
      "loss": 0.4326,
      "rewards/accuracies": 0.8125,
      "rewards/chosen": -0.3861949145793915,
      "rewards/margins": 1.226701021194458,
      "rewards/rejected": -1.6128960847854614,
      "step": 850
    },
    {
      "epoch": 0.35263003232441964,
      "grad_norm": 79.74503326416016,
      "learning_rate": 9.756509292069825e-07,
      "logits/chosen": -1.125342845916748,
      "logits/rejected": -0.875031590461731,
      "logps/chosen": -357.8549499511719,
      "logps/rejected": -289.9258117675781,
      "loss": 0.4864,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.5919533967971802,
      "rewards/margins": 1.2158567905426025,
      "rewards/rejected": -1.8078101873397827,
      "step": 900
    },
    {
      "epoch": 0.37222058967577626,
      "grad_norm": 136.9542236328125,
      "learning_rate": 9.72322864772634e-07,
      "logits/chosen": -1.1560072898864746,
      "logits/rejected": -0.886911928653717,
      "logps/chosen": -364.5694274902344,
      "logps/rejected": -293.49981689453125,
      "loss": 0.51,
      "rewards/accuracies": 0.7400000095367432,
      "rewards/chosen": -0.5705673098564148,
      "rewards/margins": 1.0413060188293457,
      "rewards/rejected": -1.6118732690811157,
      "step": 950
    },
    {
      "epoch": 0.39181114702713293,
      "grad_norm": 116.3714370727539,
      "learning_rate": 9.687881202311132e-07,
      "logits/chosen": -1.0079857110977173,
      "logits/rejected": -0.7752091884613037,
      "logps/chosen": -349.3457946777344,
      "logps/rejected": -286.99993896484375,
      "loss": 0.4722,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.4944779872894287,
      "rewards/margins": 1.1931304931640625,
      "rewards/rejected": -1.6876084804534912,
      "step": 1000
    },
    {
      "epoch": 0.39181114702713293,
      "eval_logits/chosen": -1.0761741399765015,
      "eval_logits/rejected": -0.8803514838218689,
      "eval_logps/chosen": -357.8018798828125,
      "eval_logps/rejected": -302.2410888671875,
      "eval_loss": 0.4866333603858948,
      "eval_rewards/accuracies": 0.7510408163070679,
      "eval_rewards/chosen": -0.5488451719284058,
      "eval_rewards/margins": 1.1320006847381592,
      "eval_rewards/rejected": -1.680845856666565,
      "eval_runtime": 181.5986,
      "eval_samples_per_second": 13.227,
      "eval_steps_per_second": 6.613,
      "step": 1000
    },
    {
      "epoch": 0.41140170437848955,
      "grad_norm": 169.1030731201172,
      "learning_rate": 9.65048242323929e-07,
      "logits/chosen": -1.0233768224716187,
      "logits/rejected": -0.740478515625,
      "logps/chosen": -380.3099365234375,
      "logps/rejected": -323.1108703613281,
      "loss": 0.4805,
      "rewards/accuracies": 0.7774999737739563,
      "rewards/chosen": -0.6764897704124451,
      "rewards/margins": 1.2643505334854126,
      "rewards/rejected": -1.940840482711792,
      "step": 1050
    },
    {
      "epoch": 0.4309922617298462,
      "grad_norm": 107.12033081054688,
      "learning_rate": 9.611048675553127e-07,
      "logits/chosen": -1.0799275636672974,
      "logits/rejected": -0.8886963129043579,
      "logps/chosen": -387.2933349609375,
      "logps/rejected": -321.2319030761719,
      "loss": 0.4268,
      "rewards/accuracies": 0.7774999737739563,
      "rewards/chosen": -0.5030026435852051,
      "rewards/margins": 1.4225863218307495,
      "rewards/rejected": -1.925588846206665,
      "step": 1100
    },
    {
      "epoch": 0.45058281908120285,
      "grad_norm": 124.23180389404297,
      "learning_rate": 9.569597214761124e-07,
      "logits/chosen": -1.0482264757156372,
      "logits/rejected": -0.8745830059051514,
      "logps/chosen": -352.1131591796875,
      "logps/rejected": -303.13592529296875,
      "loss": 0.4846,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": -0.689760684967041,
      "rewards/margins": 1.3204187154769897,
      "rewards/rejected": -2.0101795196533203,
      "step": 1150
    },
    {
      "epoch": 0.4701733764325595,
      "grad_norm": 68.6223373413086,
      "learning_rate": 9.526146179287222e-07,
      "logits/chosen": -1.097909927368164,
      "logits/rejected": -0.8678469657897949,
      "logps/chosen": -358.7846984863281,
      "logps/rejected": -314.0942077636719,
      "loss": 0.4829,
      "rewards/accuracies": 0.7649999856948853,
      "rewards/chosen": -0.6658291816711426,
      "rewards/margins": 1.3254380226135254,
      "rewards/rejected": -1.991267204284668,
      "step": 1200
    },
    {
      "epoch": 0.4701733764325595,
      "eval_logits/chosen": -1.1058708429336548,
      "eval_logits/rejected": -0.9207807183265686,
      "eval_logps/chosen": -358.45928955078125,
      "eval_logps/rejected": -304.12744140625,
      "eval_loss": 0.4782937467098236,
      "eval_rewards/accuracies": 0.7585345506668091,
      "eval_rewards/chosen": -0.6145861148834229,
      "eval_rewards/margins": 1.2548983097076416,
      "eval_rewards/rejected": -1.8694841861724854,
      "eval_runtime": 181.5576,
      "eval_samples_per_second": 13.23,
      "eval_steps_per_second": 6.615,
      "step": 1200
    },
    {
      "epoch": 0.48976393378391614,
      "grad_norm": 182.0519561767578,
      "learning_rate": 9.480714582533773e-07,
      "logits/chosen": -1.0906615257263184,
      "logits/rejected": -0.8988884091377258,
      "logps/chosen": -363.1963195800781,
      "logps/rejected": -310.1281433105469,
      "loss": 0.5144,
      "rewards/accuracies": 0.7200000286102295,
      "rewards/chosen": -0.7843419909477234,
      "rewards/margins": 1.230003833770752,
      "rewards/rejected": -2.014345645904541,
      "step": 1250
    },
    {
      "epoch": 0.5093544911352728,
      "grad_norm": 124.72360229492188,
      "learning_rate": 9.433322304561614e-07,
      "logits/chosen": -1.069496750831604,
      "logits/rejected": -0.8835853338241577,
      "logps/chosen": -361.09356689453125,
      "logps/rejected": -313.1136779785156,
      "loss": 0.463,
      "rewards/accuracies": 0.7825000286102295,
      "rewards/chosen": -0.662202000617981,
      "rewards/margins": 1.3414545059204102,
      "rewards/rejected": -2.0036566257476807,
      "step": 1300
    },
    {
      "epoch": 0.5289450484866295,
      "grad_norm": 124.38642120361328,
      "learning_rate": 9.383990083390903e-07,
      "logits/chosen": -1.080662488937378,
      "logits/rejected": -0.8675041794776917,
      "logps/chosen": -355.84405517578125,
      "logps/rejected": -322.5790710449219,
      "loss": 0.4625,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": -0.5591977834701538,
      "rewards/margins": 1.3175528049468994,
      "rewards/rejected": -1.8767504692077637,
      "step": 1350
    },
    {
      "epoch": 0.5485356058379861,
      "grad_norm": 110.81146240234375,
      "learning_rate": 9.332739505926528e-07,
      "logits/chosen": -1.1147152185440063,
      "logits/rejected": -0.9014944434165955,
      "logps/chosen": -360.80523681640625,
      "logps/rejected": -295.3592834472656,
      "loss": 0.4819,
      "rewards/accuracies": 0.737500011920929,
      "rewards/chosen": -0.6334275603294373,
      "rewards/margins": 1.295997142791748,
      "rewards/rejected": -1.92942476272583,
      "step": 1400
    },
    {
      "epoch": 0.5485356058379861,
      "eval_logits/chosen": -1.0615795850753784,
      "eval_logits/rejected": -0.8742749094963074,
      "eval_logps/chosen": -358.2158508300781,
      "eval_logps/rejected": -304.3434753417969,
      "eval_loss": 0.4777006208896637,
      "eval_rewards/accuracies": 0.7635303735733032,
      "eval_rewards/chosen": -0.590242862701416,
      "eval_rewards/margins": 1.3008415699005127,
      "eval_rewards/rejected": -1.8910844326019287,
      "eval_runtime": 182.5694,
      "eval_samples_per_second": 13.157,
      "eval_steps_per_second": 6.578,
      "step": 1400
    },
    {
      "epoch": 0.5681261631893427,
      "grad_norm": 99.77544403076172,
      "learning_rate": 9.279592998512064e-07,
      "logits/chosen": -1.0060768127441406,
      "logits/rejected": -0.828403115272522,
      "logps/chosen": -386.8955078125,
      "logps/rejected": -324.961669921875,
      "loss": 0.4593,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6105374693870544,
      "rewards/margins": 1.4274816513061523,
      "rewards/rejected": -2.0380191802978516,
      "step": 1450
    },
    {
      "epoch": 0.5877167205406993,
      "grad_norm": 113.63347625732422,
      "learning_rate": 9.224573817116396e-07,
      "logits/chosen": -1.1257575750350952,
      "logits/rejected": -0.8088225722312927,
      "logps/chosen": -356.369140625,
      "logps/rejected": -299.9069519042969,
      "loss": 0.4751,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": -0.7516566514968872,
      "rewards/margins": 1.3119940757751465,
      "rewards/rejected": -2.063650608062744,
      "step": 1500
    },
    {
      "epoch": 0.6073072778920561,
      "grad_norm": 71.0758285522461,
      "learning_rate": 9.16770603715733e-07,
      "logits/chosen": -1.0219570398330688,
      "logits/rejected": -0.8258044719696045,
      "logps/chosen": -365.3489074707031,
      "logps/rejected": -317.51776123046875,
      "loss": 0.4755,
      "rewards/accuracies": 0.7724999785423279,
      "rewards/chosen": -0.722920298576355,
      "rewards/margins": 1.3333240747451782,
      "rewards/rejected": -2.0562446117401123,
      "step": 1550
    },
    {
      "epoch": 0.6268978352434127,
      "grad_norm": 63.271759033203125,
      "learning_rate": 9.109014542966609e-07,
      "logits/chosen": -1.0383367538452148,
      "logits/rejected": -0.8199602365493774,
      "logps/chosen": -361.697509765625,
      "logps/rejected": -319.4376525878906,
      "loss": 0.4271,
      "rewards/accuracies": 0.8100000023841858,
      "rewards/chosen": -0.8095757961273193,
      "rewards/margins": 1.4505212306976318,
      "rewards/rejected": -2.260097026824951,
      "step": 1600
    },
    {
      "epoch": 0.6268978352434127,
      "eval_logits/chosen": -1.0107686519622803,
      "eval_logits/rejected": -0.826732873916626,
      "eval_logps/chosen": -360.0285339355469,
      "eval_logps/rejected": -307.601318359375,
      "eval_loss": 0.4775010347366333,
      "eval_rewards/accuracies": 0.768109917640686,
      "eval_rewards/chosen": -0.7715086936950684,
      "eval_rewards/margins": 1.445361852645874,
      "eval_rewards/rejected": -2.2168707847595215,
      "eval_runtime": 181.1951,
      "eval_samples_per_second": 13.256,
      "eval_steps_per_second": 6.628,
      "step": 1600
    },
    {
      "epoch": 0.6464883925947693,
      "grad_norm": 69.98014831542969,
      "learning_rate": 9.04852501690097e-07,
      "logits/chosen": -1.0106360912322998,
      "logits/rejected": -0.769228458404541,
      "logps/chosen": -361.59783935546875,
      "logps/rejected": -298.4144287109375,
      "loss": 0.4807,
      "rewards/accuracies": 0.7749999761581421,
      "rewards/chosen": -0.6844636797904968,
      "rewards/margins": 1.4386271238327026,
      "rewards/rejected": -2.1230905055999756,
      "step": 1650
    },
    {
      "epoch": 0.6660789499461259,
      "grad_norm": 73.16764831542969,
      "learning_rate": 8.986263928104007e-07,
      "logits/chosen": -1.0747100114822388,
      "logits/rejected": -0.8522970676422119,
      "logps/chosen": -377.2361755371094,
      "logps/rejected": -314.54180908203125,
      "loss": 0.475,
      "rewards/accuracies": 0.7925000190734863,
      "rewards/chosen": -0.6853989958763123,
      "rewards/margins": 1.3294869661331177,
      "rewards/rejected": -2.014885902404785,
      "step": 1700
    },
    {
      "epoch": 0.6856695072974827,
      "grad_norm": 159.627685546875,
      "learning_rate": 8.922258520923739e-07,
      "logits/chosen": -1.1866579055786133,
      "logits/rejected": -0.9568431973457336,
      "logps/chosen": -363.7584228515625,
      "logps/rejected": -335.536376953125,
      "loss": 0.4701,
      "rewards/accuracies": 0.7549999952316284,
      "rewards/chosen": -0.6905403733253479,
      "rewards/margins": 1.4639062881469727,
      "rewards/rejected": -2.1544463634490967,
      "step": 1750
    },
    {
      "epoch": 0.7052600646488393,
      "grad_norm": 59.52585983276367,
      "learning_rate": 8.856536802990969e-07,
      "logits/chosen": -1.2136218547821045,
      "logits/rejected": -0.915833055973053,
      "logps/chosen": -368.9427795410156,
      "logps/rejected": -318.7424621582031,
      "loss": 0.4355,
      "rewards/accuracies": 0.7925000190734863,
      "rewards/chosen": -0.7404943704605103,
      "rewards/margins": 1.615696907043457,
      "rewards/rejected": -2.3561911582946777,
      "step": 1800
    },
    {
      "epoch": 0.7052600646488393,
      "eval_logits/chosen": -1.137018084526062,
      "eval_logits/rejected": -0.9567646384239197,
      "eval_logps/chosen": -360.75030517578125,
      "eval_logps/rejected": -307.82464599609375,
      "eval_loss": 0.4773857891559601,
      "eval_rewards/accuracies": 0.770191490650177,
      "eval_rewards/chosen": -0.8436892032623291,
      "eval_rewards/margins": 1.3955140113830566,
      "eval_rewards/rejected": -2.2392032146453857,
      "eval_runtime": 181.4174,
      "eval_samples_per_second": 13.24,
      "eval_steps_per_second": 6.62,
      "step": 1800
    },
    {
      "epoch": 0.7248506220001959,
      "grad_norm": 145.34722900390625,
      "learning_rate": 8.789127532963639e-07,
      "logits/chosen": -1.1216288805007935,
      "logits/rejected": -0.8633120656013489,
      "logps/chosen": -379.6500244140625,
      "logps/rejected": -340.3668212890625,
      "loss": 0.4937,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.894076406955719,
      "rewards/margins": 1.335474967956543,
      "rewards/rejected": -2.2295515537261963,
      "step": 1850
    },
    {
      "epoch": 0.7444411793515525,
      "grad_norm": 85.14502716064453,
      "learning_rate": 8.720060207942547e-07,
      "logits/chosen": -1.1888411045074463,
      "logits/rejected": -0.8253584504127502,
      "logps/chosen": -358.1293029785156,
      "logps/rejected": -322.2793273925781,
      "loss": 0.453,
      "rewards/accuracies": 0.7774999737739563,
      "rewards/chosen": -0.7874532341957092,
      "rewards/margins": 1.4413965940475464,
      "rewards/rejected": -2.2288498878479004,
      "step": 1900
    },
    {
      "epoch": 0.7640317367029092,
      "grad_norm": 59.02031707763672,
      "learning_rate": 8.649365050563955e-07,
      "logits/chosen": -1.0339776277542114,
      "logits/rejected": -0.7359542846679688,
      "logps/chosen": -379.0245666503906,
      "logps/rejected": -335.61248779296875,
      "loss": 0.383,
      "rewards/accuracies": 0.824999988079071,
      "rewards/chosen": -0.9282081127166748,
      "rewards/margins": 1.724035382270813,
      "rewards/rejected": -2.6522433757781982,
      "step": 1950
    },
    {
      "epoch": 0.7836222940542659,
      "grad_norm": 100.15818786621094,
      "learning_rate": 8.577072995774679e-07,
      "logits/chosen": -1.0461410284042358,
      "logits/rejected": -0.7923344373703003,
      "logps/chosen": -358.7364196777344,
      "logps/rejected": -319.2044372558594,
      "loss": 0.5073,
      "rewards/accuracies": 0.7450000047683716,
      "rewards/chosen": -1.07187819480896,
      "rewards/margins": 1.472280740737915,
      "rewards/rejected": -2.544158697128296,
      "step": 2000
    },
    {
      "epoch": 0.7836222940542659,
      "eval_logits/chosen": -1.0298668146133423,
      "eval_logits/rejected": -0.857089638710022,
      "eval_logps/chosen": -361.0681457519531,
      "eval_logps/rejected": -309.28570556640625,
      "eval_loss": 0.4695436656475067,
      "eval_rewards/accuracies": 0.7756036520004272,
      "eval_rewards/chosen": -0.8754721283912659,
      "eval_rewards/margins": 1.5098369121551514,
      "eval_rewards/rejected": -2.3853089809417725,
      "eval_runtime": 181.4637,
      "eval_samples_per_second": 13.237,
      "eval_steps_per_second": 6.618,
      "step": 2000
    },
    {
      "epoch": 0.8032128514056225,
      "grad_norm": 109.68525695800781,
      "learning_rate": 8.503215677295522e-07,
      "logits/chosen": -1.1481313705444336,
      "logits/rejected": -0.8855399489402771,
      "logps/chosen": -377.3407897949219,
      "logps/rejected": -321.94903564453125,
      "loss": 0.4,
      "rewards/accuracies": 0.8075000047683716,
      "rewards/chosen": -0.8187921047210693,
      "rewards/margins": 1.6883336305618286,
      "rewards/rejected": -2.5071256160736084,
      "step": 2050
    },
    {
      "epoch": 0.8228034087569791,
      "grad_norm": 113.09908294677734,
      "learning_rate": 8.427825413778904e-07,
      "logits/chosen": -1.182418704032898,
      "logits/rejected": -1.0226842164993286,
      "logps/chosen": -370.88800048828125,
      "logps/rejected": -316.3885803222656,
      "loss": 0.4302,
      "rewards/accuracies": 0.7925000190734863,
      "rewards/chosen": -0.8482898473739624,
      "rewards/margins": 1.790774941444397,
      "rewards/rejected": -2.6390650272369385,
      "step": 2100
    },
    {
      "epoch": 0.8423939661083358,
      "grad_norm": 115.06123352050781,
      "learning_rate": 8.350935194666808e-07,
      "logits/chosen": -1.19704008102417,
      "logits/rejected": -0.9841734170913696,
      "logps/chosen": -344.68988037109375,
      "logps/rejected": -300.1245422363281,
      "loss": 0.4607,
      "rewards/accuracies": 0.762499988079071,
      "rewards/chosen": -0.8847514390945435,
      "rewards/margins": 1.6138572692871094,
      "rewards/rejected": -2.4986085891723633,
      "step": 2150
    },
    {
      "epoch": 0.8619845234596925,
      "grad_norm": 91.62993621826172,
      "learning_rate": 8.272578665755176e-07,
      "logits/chosen": -1.1561052799224854,
      "logits/rejected": -0.892173171043396,
      "logps/chosen": -357.5077819824219,
      "logps/rejected": -313.1963806152344,
      "loss": 0.4934,
      "rewards/accuracies": 0.7574999928474426,
      "rewards/chosen": -0.9803217053413391,
      "rewards/margins": 1.5475577116012573,
      "rewards/rejected": -2.527879238128662,
      "step": 2200
    },
    {
      "epoch": 0.8619845234596925,
      "eval_logits/chosen": -1.0830334424972534,
      "eval_logits/rejected": -0.9147012233734131,
      "eval_logps/chosen": -361.2430114746094,
      "eval_logps/rejected": -310.0792541503906,
      "eval_loss": 0.4646490514278412,
      "eval_rewards/accuracies": 0.7768526077270508,
      "eval_rewards/chosen": -0.8929603099822998,
      "eval_rewards/margins": 1.5717030763626099,
      "eval_rewards/rejected": -2.464663505554199,
      "eval_runtime": 183.2831,
      "eval_samples_per_second": 13.105,
      "eval_steps_per_second": 6.553,
      "step": 2200
    },
    {
      "epoch": 0.8815750808110491,
      "grad_norm": 72.61209869384766,
      "learning_rate": 8.192790114471106e-07,
      "logits/chosen": -1.1451025009155273,
      "logits/rejected": -0.9403939247131348,
      "logps/chosen": -370.800048828125,
      "logps/rejected": -303.31817626953125,
      "loss": 0.4352,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.7773234844207764,
      "rewards/margins": 1.5500972270965576,
      "rewards/rejected": -2.327420711517334,
      "step": 2250
    },
    {
      "epoch": 0.9011656381624057,
      "grad_norm": 70.87027740478516,
      "learning_rate": 8.111604454869285e-07,
      "logits/chosen": -1.17734956741333,
      "logits/rejected": -0.9346526861190796,
      "logps/chosen": -345.1195068359375,
      "logps/rejected": -306.4920654296875,
      "loss": 0.4703,
      "rewards/accuracies": 0.7674999833106995,
      "rewards/chosen": -0.9427626729011536,
      "rewards/margins": 1.593432903289795,
      "rewards/rejected": -2.536195755004883,
      "step": 2300
    },
    {
      "epoch": 0.9207561955137624,
      "grad_norm": 90.32552337646484,
      "learning_rate": 8.029057212354218e-07,
      "logits/chosen": -1.0256972312927246,
      "logits/rejected": -0.9072439074516296,
      "logps/chosen": -364.8022155761719,
      "logps/rejected": -313.5185241699219,
      "loss": 0.4846,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.1208609342575073,
      "rewards/margins": 1.5102592706680298,
      "rewards/rejected": -2.631119966506958,
      "step": 2350
    },
    {
      "epoch": 0.940346752865119,
      "grad_norm": 143.2118377685547,
      "learning_rate": 7.945184508134936e-07,
      "logits/chosen": -1.1666345596313477,
      "logits/rejected": -0.9767065644264221,
      "logps/chosen": -391.8117980957031,
      "logps/rejected": -319.5590515136719,
      "loss": 0.4403,
      "rewards/accuracies": 0.7875000238418579,
      "rewards/chosen": -0.9298499822616577,
      "rewards/margins": 1.5455856323242188,
      "rewards/rejected": -2.475435495376587,
      "step": 2400
    },
    {
      "epoch": 0.940346752865119,
      "eval_logits/chosen": -1.10908043384552,
      "eval_logits/rejected": -0.9499452710151672,
      "eval_logps/chosen": -362.099609375,
      "eval_logps/rejected": -311.1372375488281,
      "eval_loss": 0.46884483098983765,
      "eval_rewards/accuracies": 0.7722731232643127,
      "eval_rewards/chosen": -0.9786169528961182,
      "eval_rewards/margins": 1.5918443202972412,
      "eval_rewards/rejected": -2.5704612731933594,
      "eval_runtime": 181.3567,
      "eval_samples_per_second": 13.245,
      "eval_steps_per_second": 6.622,
      "step": 2400
    },
    {
      "epoch": 0.9599373102164757,
      "grad_norm": 192.61532592773438,
      "learning_rate": 7.860023043419004e-07,
      "logits/chosen": -1.1775944232940674,
      "logits/rejected": -0.9322341084480286,
      "logps/chosen": -363.9286193847656,
      "logps/rejected": -324.30621337890625,
      "loss": 0.4438,
      "rewards/accuracies": 0.8174999952316284,
      "rewards/chosen": -1.010249376296997,
      "rewards/margins": 1.8364521265029907,
      "rewards/rejected": -2.8467013835906982,
      "step": 2450
    },
    {
      "epoch": 0.9795278675678323,
      "grad_norm": 154.35006713867188,
      "learning_rate": 7.773610083352717e-07,
      "logits/chosen": -0.9925041794776917,
      "logits/rejected": -0.813052773475647,
      "logps/chosen": -363.6401062011719,
      "logps/rejected": -323.7989501953125,
      "loss": 0.5359,
      "rewards/accuracies": 0.7225000262260437,
      "rewards/chosen": -1.0799657106399536,
      "rewards/margins": 1.3734463453292847,
      "rewards/rejected": -2.453411817550659,
      "step": 2500
    },
    {
      "epoch": 0.9991184249191889,
      "grad_norm": 196.2805633544922,
      "learning_rate": 7.685983440714535e-07,
      "logits/chosen": -1.093634009361267,
      "logits/rejected": -0.8999694585800171,
      "logps/chosen": -393.096435546875,
      "logps/rejected": -340.03387451171875,
      "loss": 0.4938,
      "rewards/accuracies": 0.7699999809265137,
      "rewards/chosen": -1.0175117254257202,
      "rewards/margins": 1.5324511528015137,
      "rewards/rejected": -2.5499629974365234,
      "step": 2550
    },
    {
      "epoch": 1.0184151239102752,
      "grad_norm": 34.11044692993164,
      "learning_rate": 7.597181459368884e-07,
      "logits/chosen": -1.1935698986053467,
      "logits/rejected": -0.9796786308288574,
      "logps/chosen": -363.0191955566406,
      "logps/rejected": -347.12249755859375,
      "loss": 0.129,
      "rewards/accuracies": 0.9670050740242004,
      "rewards/chosen": -0.2811751663684845,
      "rewards/margins": 3.3759641647338867,
      "rewards/rejected": -3.657139301300049,
      "step": 2600
    },
    {
      "epoch": 1.0184151239102752,
      "eval_logits/chosen": -1.2456655502319336,
      "eval_logits/rejected": -1.0991578102111816,
      "eval_logps/chosen": -363.8660888671875,
      "eval_logps/rejected": -313.871826171875,
      "eval_loss": 0.47346433997154236,
      "eval_rewards/accuracies": 0.7718567848205566,
      "eval_rewards/chosen": -1.1552621126174927,
      "eval_rewards/margins": 1.68865966796875,
      "eval_rewards/rejected": -2.8439218997955322,
      "eval_runtime": 181.5327,
      "eval_samples_per_second": 13.232,
      "eval_steps_per_second": 6.616,
      "step": 2600
    },
    {
      "epoch": 1.0380056812616318,
      "grad_norm": 75.25180053710938,
      "learning_rate": 7.50724299748756e-07,
      "logits/chosen": -1.2610849142074585,
      "logits/rejected": -1.150437355041504,
      "logps/chosen": -375.75,
      "logps/rejected": -315.3388366699219,
      "loss": 0.146,
      "rewards/accuracies": 0.9674999713897705,
      "rewards/chosen": -0.38413771986961365,
      "rewards/margins": 3.210354804992676,
      "rewards/rejected": -3.5944931507110596,
      "step": 2650
    },
    {
      "epoch": 1.0575962386129885,
      "grad_norm": 19.142627716064453,
      "learning_rate": 7.416207410546075e-07,
      "logits/chosen": -1.4331450462341309,
      "logits/rejected": -1.1033505201339722,
      "logps/chosen": -341.708251953125,
      "logps/rejected": -322.0888977050781,
      "loss": 0.1627,
      "rewards/accuracies": 0.9449999928474426,
      "rewards/chosen": -0.5315707921981812,
      "rewards/margins": 3.344498872756958,
      "rewards/rejected": -3.876070022583008,
      "step": 2700
    },
    {
      "epoch": 1.0771867959643453,
      "grad_norm": 18.423038482666016,
      "learning_rate": 7.324114534102414e-07,
      "logits/chosen": -1.2948585748672485,
      "logits/rejected": -1.1851221323013306,
      "logps/chosen": -384.01495361328125,
      "logps/rejected": -340.6744384765625,
      "loss": 0.1212,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -0.458404541015625,
      "rewards/margins": 3.760544776916504,
      "rewards/rejected": -4.218949317932129,
      "step": 2750
    },
    {
      "epoch": 1.096777353315702,
      "grad_norm": 15.992474555969238,
      "learning_rate": 7.231004666365688e-07,
      "logits/chosen": -1.5737820863723755,
      "logits/rejected": -1.3861126899719238,
      "logps/chosen": -370.8525695800781,
      "logps/rejected": -342.98675537109375,
      "loss": 0.1362,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.7288404703140259,
      "rewards/margins": 3.653024911880493,
      "rewards/rejected": -4.381865978240967,
      "step": 2800
    },
    {
      "epoch": 1.096777353315702,
      "eval_logits/chosen": -1.4936131238937378,
      "eval_logits/rejected": -1.3645743131637573,
      "eval_logps/chosen": -369.6283874511719,
      "eval_logps/rejected": -322.2799072265625,
      "eval_loss": 0.5014060735702515,
      "eval_rewards/accuracies": 0.7772689461708069,
      "eval_rewards/chosen": -1.7314954996109009,
      "eval_rewards/margins": 1.9532350301742554,
      "eval_rewards/rejected": -3.6847305297851562,
      "eval_runtime": 183.7133,
      "eval_samples_per_second": 13.075,
      "eval_steps_per_second": 6.537,
      "step": 2800
    },
    {
      "epoch": 1.1163679106670585,
      "grad_norm": 5.409101963043213,
      "learning_rate": 7.136918550562358e-07,
      "logits/chosen": -1.522940993309021,
      "logits/rejected": -1.4030097723007202,
      "logps/chosen": -366.8924560546875,
      "logps/rejected": -333.6314697265625,
      "loss": 0.1492,
      "rewards/accuracies": 0.9449999928474426,
      "rewards/chosen": -0.8802686929702759,
      "rewards/margins": 3.616269826889038,
      "rewards/rejected": -4.496538162231445,
      "step": 2850
    },
    {
      "epoch": 1.1359584680184152,
      "grad_norm": 54.6688232421875,
      "learning_rate": 7.041897357107727e-07,
      "logits/chosen": -1.6211928129196167,
      "logits/rejected": -1.4868953227996826,
      "logps/chosen": -359.833740234375,
      "logps/rejected": -310.93939208984375,
      "loss": 0.1698,
      "rewards/accuracies": 0.9424999952316284,
      "rewards/chosen": -0.7522315979003906,
      "rewards/margins": 3.544074058532715,
      "rewards/rejected": -4.296305179595947,
      "step": 2900
    },
    {
      "epoch": 1.1555490253697718,
      "grad_norm": 56.798030853271484,
      "learning_rate": 6.945982665590479e-07,
      "logits/chosen": -1.4974216222763062,
      "logits/rejected": -1.2985191345214844,
      "logps/chosen": -377.2979736328125,
      "logps/rejected": -348.5942077636719,
      "loss": 0.146,
      "rewards/accuracies": 0.9399999976158142,
      "rewards/chosen": -0.8809213042259216,
      "rewards/margins": 3.612612247467041,
      "rewards/rejected": -4.493533134460449,
      "step": 2950
    },
    {
      "epoch": 1.1751395827211284,
      "grad_norm": 22.727725982666016,
      "learning_rate": 6.849216446578215e-07,
      "logits/chosen": -1.590391993522644,
      "logits/rejected": -1.433569073677063,
      "logps/chosen": -387.9193115234375,
      "logps/rejected": -361.5350646972656,
      "loss": 0.1149,
      "rewards/accuracies": 0.9725000262260437,
      "rewards/chosen": -1.0071202516555786,
      "rewards/margins": 3.8817479610443115,
      "rewards/rejected": -4.88886833190918,
      "step": 3000
    },
    {
      "epoch": 1.1751395827211284,
      "eval_logits/chosen": -1.6452481746673584,
      "eval_logits/rejected": -1.5406179428100586,
      "eval_logps/chosen": -371.4642333984375,
      "eval_logps/rejected": -325.0046691894531,
      "eval_loss": 0.5132338404655457,
      "eval_rewards/accuracies": 0.770191490650177,
      "eval_rewards/chosen": -1.9150793552398682,
      "eval_rewards/margins": 2.042127847671509,
      "eval_rewards/rejected": -3.957206964492798,
      "eval_runtime": 182.0948,
      "eval_samples_per_second": 13.191,
      "eval_steps_per_second": 6.595,
      "step": 3000
    },
    {
      "epoch": 1.194730140072485,
      "grad_norm": 25.459936141967773,
      "learning_rate": 6.751641043251853e-07,
      "logits/chosen": -1.5482993125915527,
      "logits/rejected": -1.4024869203567505,
      "logps/chosen": -392.9327697753906,
      "logps/rejected": -338.0341796875,
      "loss": 0.1499,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -0.9234389662742615,
      "rewards/margins": 3.838209629058838,
      "rewards/rejected": -4.761648178100586,
      "step": 3050
    },
    {
      "epoch": 1.2143206974238416,
      "grad_norm": 71.29440307617188,
      "learning_rate": 6.653299152877016e-07,
      "logits/chosen": -1.7307244539260864,
      "logits/rejected": -1.550933837890625,
      "logps/chosen": -357.46661376953125,
      "logps/rejected": -342.1903076171875,
      "loss": 0.1345,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -1.0262528657913208,
      "rewards/margins": 3.8201754093170166,
      "rewards/rejected": -4.846428871154785,
      "step": 3100
    },
    {
      "epoch": 1.2339112547751983,
      "grad_norm": 25.778587341308594,
      "learning_rate": 6.554233808120463e-07,
      "logits/chosen": -1.6679961681365967,
      "logits/rejected": -1.5830594301223755,
      "logps/chosen": -386.9595642089844,
      "logps/rejected": -344.4521789550781,
      "loss": 0.1438,
      "rewards/accuracies": 0.9649999737739563,
      "rewards/chosen": -1.1483721733093262,
      "rewards/margins": 3.911073684692383,
      "rewards/rejected": -5.059445858001709,
      "step": 3150
    },
    {
      "epoch": 1.2535018121265549,
      "grad_norm": 24.407367706298828,
      "learning_rate": 6.454488358219756e-07,
      "logits/chosen": -1.6095058917999268,
      "logits/rejected": -1.5268667936325073,
      "logps/chosen": -364.7217102050781,
      "logps/rejected": -321.9256591796875,
      "loss": 0.1198,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -0.9407815337181091,
      "rewards/margins": 3.9222023487091064,
      "rewards/rejected": -4.862983703613281,
      "step": 3200
    },
    {
      "epoch": 1.2535018121265549,
      "eval_logits/chosen": -1.6486918926239014,
      "eval_logits/rejected": -1.56648588180542,
      "eval_logps/chosen": -373.4631652832031,
      "eval_logps/rejected": -328.3328552246094,
      "eval_loss": 0.5195496678352356,
      "eval_rewards/accuracies": 0.7751873731613159,
      "eval_rewards/chosen": -2.114971399307251,
      "eval_rewards/margins": 2.175050735473633,
      "eval_rewards/rejected": -4.290021896362305,
      "eval_runtime": 182.0558,
      "eval_samples_per_second": 13.194,
      "eval_steps_per_second": 6.597,
      "step": 3200
    },
    {
      "epoch": 1.2730923694779117,
      "grad_norm": 61.75260925292969,
      "learning_rate": 6.354106450014404e-07,
      "logits/chosen": -1.6261045932769775,
      "logits/rejected": -1.468245267868042,
      "logps/chosen": -394.1415100097656,
      "logps/rejected": -353.34942626953125,
      "loss": 0.1666,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -1.1136356592178345,
      "rewards/margins": 4.16200065612793,
      "rewards/rejected": -5.275636196136475,
      "step": 3250
    },
    {
      "epoch": 1.2926829268292683,
      "grad_norm": 19.361291885375977,
      "learning_rate": 6.253132008846786e-07,
      "logits/chosen": -1.6353635787963867,
      "logits/rejected": -1.5276310443878174,
      "logps/chosen": -366.8433532714844,
      "logps/rejected": -341.7208251953125,
      "loss": 0.1418,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -0.9791030883789062,
      "rewards/margins": 3.893622636795044,
      "rewards/rejected": -4.872725963592529,
      "step": 3300
    },
    {
      "epoch": 1.312273484180625,
      "grad_norm": 11.505748748779297,
      "learning_rate": 6.15160921934118e-07,
      "logits/chosen": -1.6867657899856567,
      "logits/rejected": -1.5593910217285156,
      "logps/chosen": -365.0918884277344,
      "logps/rejected": -318.346923828125,
      "loss": 0.1254,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -0.9864305257797241,
      "rewards/margins": 3.980782985687256,
      "rewards/rejected": -4.9672136306762695,
      "step": 3350
    },
    {
      "epoch": 1.3318640415319816,
      "grad_norm": 48.97773742675781,
      "learning_rate": 6.049582506069384e-07,
      "logits/chosen": -1.657313585281372,
      "logits/rejected": -1.4994758367538452,
      "logps/chosen": -359.9202880859375,
      "logps/rejected": -347.42828369140625,
      "loss": 0.1331,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.2405058145523071,
      "rewards/margins": 3.8140506744384766,
      "rewards/rejected": -5.054556369781494,
      "step": 3400
    },
    {
      "epoch": 1.3318640415319816,
      "eval_logits/chosen": -1.716418981552124,
      "eval_logits/rejected": -1.6371155977249146,
      "eval_logps/chosen": -372.63067626953125,
      "eval_logps/rejected": -327.14337158203125,
      "eval_loss": 0.5230005979537964,
      "eval_rewards/accuracies": 0.7793505191802979,
      "eval_rewards/chosen": -2.0317211151123047,
      "eval_rewards/margins": 2.1393544673919678,
      "eval_rewards/rejected": -4.17107629776001,
      "eval_runtime": 182.2547,
      "eval_samples_per_second": 13.179,
      "eval_steps_per_second": 6.59,
      "step": 3400
    },
    {
      "epoch": 1.3514545988833382,
      "grad_norm": 107.73078155517578,
      "learning_rate": 5.947096514111293e-07,
      "logits/chosen": -1.6291078329086304,
      "logits/rejected": -1.5198391675949097,
      "logps/chosen": -383.50958251953125,
      "logps/rejected": -363.100341796875,
      "loss": 0.1277,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.0857621431350708,
      "rewards/margins": 4.09164571762085,
      "rewards/rejected": -5.177408218383789,
      "step": 3450
    },
    {
      "epoch": 1.3710451562346948,
      "grad_norm": 15.163590431213379,
      "learning_rate": 5.844196089519004e-07,
      "logits/chosen": -1.6963342428207397,
      "logits/rejected": -1.5234904289245605,
      "logps/chosen": -367.5895690917969,
      "logps/rejected": -334.65203857421875,
      "loss": 0.1338,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -1.2845149040222168,
      "rewards/margins": 4.080843448638916,
      "rewards/rejected": -5.365358829498291,
      "step": 3500
    },
    {
      "epoch": 1.3906357135860516,
      "grad_norm": 183.16290283203125,
      "learning_rate": 5.740926259692986e-07,
      "logits/chosen": -1.6925588846206665,
      "logits/rejected": -1.4511497020721436,
      "logps/chosen": -342.0213317871094,
      "logps/rejected": -328.423583984375,
      "loss": 0.1684,
      "rewards/accuracies": 0.9150000214576721,
      "rewards/chosen": -1.2314543724060059,
      "rewards/margins": 4.002154350280762,
      "rewards/rejected": -5.233608245849609,
      "step": 3550
    },
    {
      "epoch": 1.4102262709374083,
      "grad_norm": 130.83932495117188,
      "learning_rate": 5.637332213678889e-07,
      "logits/chosen": -1.747999668121338,
      "logits/rejected": -1.552089810371399,
      "logps/chosen": -357.76312255859375,
      "logps/rejected": -321.6129150390625,
      "loss": 0.1506,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -1.2750509977340698,
      "rewards/margins": 4.070872783660889,
      "rewards/rejected": -5.345923900604248,
      "step": 3600
    },
    {
      "epoch": 1.4102262709374083,
      "eval_logits/chosen": -1.6983522176742554,
      "eval_logits/rejected": -1.6196881532669067,
      "eval_logps/chosen": -374.8834228515625,
      "eval_logps/rejected": -329.9042053222656,
      "eval_loss": 0.5274552702903748,
      "eval_rewards/accuracies": 0.7756036520004272,
      "eval_rewards/chosen": -2.256995916366577,
      "eval_rewards/margins": 2.1901612281799316,
      "eval_rewards/rejected": -4.4471564292907715,
      "eval_runtime": 182.0012,
      "eval_samples_per_second": 13.198,
      "eval_steps_per_second": 6.599,
      "step": 3600
    },
    {
      "epoch": 1.4298168282887649,
      "grad_norm": 62.9024658203125,
      "learning_rate": 5.53345928239361e-07,
      "logits/chosen": -1.6344012022018433,
      "logits/rejected": -1.3985601663589478,
      "logps/chosen": -371.9190979003906,
      "logps/rejected": -362.4126892089844,
      "loss": 0.1324,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -1.3249839544296265,
      "rewards/margins": 4.2258501052856445,
      "rewards/rejected": -5.550833702087402,
      "step": 3650
    },
    {
      "epoch": 1.4494073856401215,
      "grad_norm": 63.766395568847656,
      "learning_rate": 5.429352918789295e-07,
      "logits/chosen": -1.6781551837921143,
      "logits/rejected": -1.6595940589904785,
      "logps/chosen": -359.553955078125,
      "logps/rejected": -333.27191162109375,
      "loss": 0.1255,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -1.3423984050750732,
      "rewards/margins": 4.154109477996826,
      "rewards/rejected": -5.496507167816162,
      "step": 3700
    },
    {
      "epoch": 1.4689979429914781,
      "grad_norm": 24.34297752380371,
      "learning_rate": 5.325058677963933e-07,
      "logits/chosen": -1.7628166675567627,
      "logits/rejected": -1.5162057876586914,
      "logps/chosen": -379.5002746582031,
      "logps/rejected": -361.4959411621094,
      "loss": 0.1278,
      "rewards/accuracies": 0.9424999952316284,
      "rewards/chosen": -1.2505874633789062,
      "rewards/margins": 4.234759330749512,
      "rewards/rejected": -5.485346794128418,
      "step": 3750
    },
    {
      "epoch": 1.4885885003428347,
      "grad_norm": 89.56745910644531,
      "learning_rate": 5.220622197227254e-07,
      "logits/chosen": -1.6744155883789062,
      "logits/rejected": -1.6061152219772339,
      "logps/chosen": -395.9300842285156,
      "logps/rejected": -356.6335144042969,
      "loss": 0.1219,
      "rewards/accuracies": 0.9474999904632568,
      "rewards/chosen": -1.2993007898330688,
      "rewards/margins": 4.109463214874268,
      "rewards/rejected": -5.408763885498047,
      "step": 3800
    },
    {
      "epoch": 1.4885885003428347,
      "eval_logits/chosen": -1.755277156829834,
      "eval_logits/rejected": -1.6812604665756226,
      "eval_logps/chosen": -375.90478515625,
      "eval_logps/rejected": -331.6509704589844,
      "eval_loss": 0.5298057794570923,
      "eval_rewards/accuracies": 0.7793505191802979,
      "eval_rewards/chosen": -2.3591370582580566,
      "eval_rewards/margins": 2.262698173522949,
      "eval_rewards/rejected": -4.621835708618164,
      "eval_runtime": 181.8002,
      "eval_samples_per_second": 13.212,
      "eval_steps_per_second": 6.606,
      "step": 3800
    },
    {
      "epoch": 1.5081790576941914,
      "grad_norm": 57.11545944213867,
      "learning_rate": 5.116089176130647e-07,
      "logits/chosen": -1.8247219324111938,
      "logits/rejected": -1.707238793373108,
      "logps/chosen": -365.5677795410156,
      "logps/rejected": -348.8277587890625,
      "loss": 0.1248,
      "rewards/accuracies": 0.9649999737739563,
      "rewards/chosen": -1.4670463800430298,
      "rewards/margins": 4.245115756988525,
      "rewards/rejected": -5.712162017822266,
      "step": 3850
    },
    {
      "epoch": 1.527769615045548,
      "grad_norm": 134.8478546142578,
      "learning_rate": 5.011505356469849e-07,
      "logits/chosen": -1.800379991531372,
      "logits/rejected": -1.6222047805786133,
      "logps/chosen": -396.044677734375,
      "logps/rejected": -343.588623046875,
      "loss": 0.1579,
      "rewards/accuracies": 0.9350000023841858,
      "rewards/chosen": -1.327430248260498,
      "rewards/margins": 4.10135555267334,
      "rewards/rejected": -5.428785800933838,
      "step": 3900
    },
    {
      "epoch": 1.5473601723969046,
      "grad_norm": 47.98903274536133,
      "learning_rate": 4.906916502269153e-07,
      "logits/chosen": -1.8289942741394043,
      "logits/rejected": -1.7046104669570923,
      "logps/chosen": -360.898681640625,
      "logps/rejected": -347.4443359375,
      "loss": 0.1689,
      "rewards/accuracies": 0.9424999952316284,
      "rewards/chosen": -1.565004825592041,
      "rewards/margins": 4.049954414367676,
      "rewards/rejected": -5.6149582862854,
      "step": 3950
    },
    {
      "epoch": 1.5669507297482612,
      "grad_norm": 36.070865631103516,
      "learning_rate": 4.802368379755871e-07,
      "logits/chosen": -1.901904582977295,
      "logits/rejected": -1.7009533643722534,
      "logps/chosen": -390.9181213378906,
      "logps/rejected": -367.340087890625,
      "loss": 0.1173,
      "rewards/accuracies": 0.9649999737739563,
      "rewards/chosen": -1.3107359409332275,
      "rewards/margins": 4.3210906982421875,
      "rewards/rejected": -5.631826877593994,
      "step": 4000
    },
    {
      "epoch": 1.5669507297482612,
      "eval_logits/chosen": -1.83998441696167,
      "eval_logits/rejected": -1.773179292678833,
      "eval_logps/chosen": -375.8030700683594,
      "eval_logps/rejected": -331.4768371582031,
      "eval_loss": 0.5410642027854919,
      "eval_rewards/accuracies": 0.779766857624054,
      "eval_rewards/chosen": -2.348963737487793,
      "eval_rewards/margins": 2.255459785461426,
      "eval_rewards/rejected": -4.604423522949219,
      "eval_runtime": 182.1217,
      "eval_samples_per_second": 13.189,
      "eval_steps_per_second": 6.594,
      "step": 4000
    },
    {
      "epoch": 1.5865412870996178,
      "grad_norm": 79.75633239746094,
      "learning_rate": 4.697906737333856e-07,
      "logits/chosen": -1.890414834022522,
      "logits/rejected": -1.7079460620880127,
      "logps/chosen": -363.3149108886719,
      "logps/rejected": -336.81964111328125,
      "loss": 0.1136,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -1.327339768409729,
      "rewards/margins": 4.425736904144287,
      "rewards/rejected": -5.753076553344727,
      "step": 4050
    },
    {
      "epoch": 1.6061318444509747,
      "grad_norm": 14.279524803161621,
      "learning_rate": 4.593577285564805e-07,
      "logits/chosen": -1.9139132499694824,
      "logits/rejected": -1.7257100343704224,
      "logps/chosen": -373.6604309082031,
      "logps/rejected": -360.3436279296875,
      "loss": 0.1217,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -1.4224177598953247,
      "rewards/margins": 4.337459564208984,
      "rewards/rejected": -5.759876728057861,
      "step": 4100
    },
    {
      "epoch": 1.6257224018023313,
      "grad_norm": 18.292810440063477,
      "learning_rate": 4.489425677166128e-07,
      "logits/chosen": -1.8857932090759277,
      "logits/rejected": -1.838065505027771,
      "logps/chosen": -374.2807922363281,
      "logps/rejected": -350.9076232910156,
      "loss": 0.1116,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -1.2832770347595215,
      "rewards/margins": 4.261753082275391,
      "rewards/rejected": -5.545030117034912,
      "step": 4150
    },
    {
      "epoch": 1.645312959153688,
      "grad_norm": 91.74571990966797,
      "learning_rate": 4.385497487034136e-07,
      "logits/chosen": -1.8253960609436035,
      "logits/rejected": -1.80033540725708,
      "logps/chosen": -360.9422607421875,
      "logps/rejected": -348.8204040527344,
      "loss": 0.1668,
      "rewards/accuracies": 0.9449999928474426,
      "rewards/chosen": -1.5318783521652222,
      "rewards/margins": 4.232841968536377,
      "rewards/rejected": -5.764720439910889,
      "step": 4200
    },
    {
      "epoch": 1.645312959153688,
      "eval_logits/chosen": -1.825873851776123,
      "eval_logits/rejected": -1.7672475576400757,
      "eval_logps/chosen": -375.93621826171875,
      "eval_logps/rejected": -332.0399475097656,
      "eval_loss": 0.5490909814834595,
      "eval_rewards/accuracies": 0.7781015634536743,
      "eval_rewards/chosen": -2.362279176712036,
      "eval_rewards/margins": 2.2984535694122314,
      "eval_rewards/rejected": -4.660732746124268,
      "eval_runtime": 181.3571,
      "eval_samples_per_second": 13.245,
      "eval_steps_per_second": 6.622,
      "step": 4200
    },
    {
      "epoch": 1.6649035165050445,
      "grad_norm": 182.9907989501953,
      "learning_rate": 4.281838192301266e-07,
      "logits/chosen": -1.8667634725570679,
      "logits/rejected": -1.7775866985321045,
      "logps/chosen": -350.6466064453125,
      "logps/rejected": -349.0397644042969,
      "loss": 0.133,
      "rewards/accuracies": 0.9424999952316284,
      "rewards/chosen": -1.5763401985168457,
      "rewards/margins": 4.427700996398926,
      "rewards/rejected": -6.004040718078613,
      "step": 4250
    },
    {
      "epoch": 1.6844940738564014,
      "grad_norm": 90.31336975097656,
      "learning_rate": 4.1784931524360996e-07,
      "logits/chosen": -1.8399044275283813,
      "logits/rejected": -1.7566088438034058,
      "logps/chosen": -357.8070983886719,
      "logps/rejected": -337.547119140625,
      "loss": 0.1338,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -1.5925673246383667,
      "rewards/margins": 4.302795886993408,
      "rewards/rejected": -5.895363807678223,
      "step": 4300
    },
    {
      "epoch": 1.704084631207758,
      "grad_norm": 60.01850128173828,
      "learning_rate": 4.075507589394862e-07,
      "logits/chosen": -1.8707722425460815,
      "logits/rejected": -1.7477329969406128,
      "logps/chosen": -367.755126953125,
      "logps/rejected": -334.7363586425781,
      "loss": 0.1432,
      "rewards/accuracies": 0.9399999976158142,
      "rewards/chosen": -1.4867920875549316,
      "rewards/margins": 4.018145561218262,
      "rewards/rejected": -5.504937171936035,
      "step": 4350
    },
    {
      "epoch": 1.7236751885591146,
      "grad_norm": 7.138427257537842,
      "learning_rate": 3.972926567833095e-07,
      "logits/chosen": -1.7986953258514404,
      "logits/rejected": -1.716946005821228,
      "logps/chosen": -377.9021911621094,
      "logps/rejected": -367.26483154296875,
      "loss": 0.1184,
      "rewards/accuracies": 0.949999988079071,
      "rewards/chosen": -1.5053943395614624,
      "rewards/margins": 4.413090705871582,
      "rewards/rejected": -5.918485164642334,
      "step": 4400
    },
    {
      "epoch": 1.7236751885591146,
      "eval_logits/chosen": -1.7733187675476074,
      "eval_logits/rejected": -1.7134273052215576,
      "eval_logps/chosen": -377.1297607421875,
      "eval_logps/rejected": -333.46954345703125,
      "eval_loss": 0.5378764271736145,
      "eval_rewards/accuracies": 0.7814321517944336,
      "eval_rewards/chosen": -2.4816348552703857,
      "eval_rewards/margins": 2.3220555782318115,
      "eval_rewards/rejected": -4.803690433502197,
      "eval_runtime": 182.8904,
      "eval_samples_per_second": 13.134,
      "eval_steps_per_second": 6.567,
      "step": 4400
    },
    {
      "epoch": 1.7432657459104712,
      "grad_norm": 18.62919807434082,
      "learning_rate": 3.8707949753861716e-07,
      "logits/chosen": -1.8632616996765137,
      "logits/rejected": -1.7376039028167725,
      "logps/chosen": -342.5689697265625,
      "logps/rejected": -335.2397766113281,
      "loss": 0.1411,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -1.3847324848175049,
      "rewards/margins": 4.106419563293457,
      "rewards/rejected": -5.491152286529541,
      "step": 4450
    },
    {
      "epoch": 1.7628563032618279,
      "grad_norm": 23.290203094482422,
      "learning_rate": 3.76915750302725e-07,
      "logits/chosen": -1.7581045627593994,
      "logits/rejected": -1.6029043197631836,
      "logps/chosen": -373.6942138671875,
      "logps/rejected": -338.0625,
      "loss": 0.1141,
      "rewards/accuracies": 0.9649999737739563,
      "rewards/chosen": -1.4231642484664917,
      "rewards/margins": 4.145116806030273,
      "rewards/rejected": -5.5682806968688965,
      "step": 4500
    },
    {
      "epoch": 1.7824468606131845,
      "grad_norm": 113.75978088378906,
      "learning_rate": 3.668058625511305e-07,
      "logits/chosen": -1.7640550136566162,
      "logits/rejected": -1.6161187887191772,
      "logps/chosen": -369.67816162109375,
      "logps/rejected": -363.20465087890625,
      "loss": 0.1117,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -1.3803114891052246,
      "rewards/margins": 4.385107040405273,
      "rewards/rejected": -5.765418529510498,
      "step": 4550
    },
    {
      "epoch": 1.802037417964541,
      "grad_norm": 7.093522548675537,
      "learning_rate": 3.567542581913762e-07,
      "logits/chosen": -1.8656272888183594,
      "logits/rejected": -1.8022925853729248,
      "logps/chosen": -395.6029968261719,
      "logps/rejected": -359.8951110839844,
      "loss": 0.1466,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -1.3073208332061768,
      "rewards/margins": 4.359652519226074,
      "rewards/rejected": -5.666974067687988,
      "step": 4600
    },
    {
      "epoch": 1.802037417964541,
      "eval_logits/chosen": -1.739977240562439,
      "eval_logits/rejected": -1.677179217338562,
      "eval_logps/chosen": -376.30279541015625,
      "eval_logps/rejected": -332.7083740234375,
      "eval_loss": 0.5302212834358215,
      "eval_rewards/accuracies": 0.7843464016914368,
      "eval_rewards/chosen": -2.3989357948303223,
      "eval_rewards/margins": 2.3286385536193848,
      "eval_rewards/rejected": -4.727574348449707,
      "eval_runtime": 181.7513,
      "eval_samples_per_second": 13.216,
      "eval_steps_per_second": 6.608,
      "step": 4600
    },
    {
      "epoch": 1.8216279753158977,
      "grad_norm": 168.2427978515625,
      "learning_rate": 3.467653356272264e-07,
      "logits/chosen": -1.7600982189178467,
      "logits/rejected": -1.75065016746521,
      "logps/chosen": -368.1263732910156,
      "logps/rejected": -329.2120056152344,
      "loss": 0.1393,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -1.325424313545227,
      "rewards/margins": 4.229908466339111,
      "rewards/rejected": -5.555332660675049,
      "step": 4650
    },
    {
      "epoch": 1.8412185326672543,
      "grad_norm": 56.983543395996094,
      "learning_rate": 3.368434658340035e-07,
      "logits/chosen": -1.6969270706176758,
      "logits/rejected": -1.5315319299697876,
      "logps/chosen": -393.845947265625,
      "logps/rejected": -366.9394836425781,
      "loss": 0.1201,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -1.3427032232284546,
      "rewards/margins": 4.412410259246826,
      "rewards/rejected": -5.755113124847412,
      "step": 4700
    },
    {
      "epoch": 1.860809090018611,
      "grad_norm": 46.993534088134766,
      "learning_rate": 3.269929904459265e-07,
      "logits/chosen": -1.6990869045257568,
      "logits/rejected": -1.6050549745559692,
      "logps/chosen": -379.8760070800781,
      "logps/rejected": -358.8374938964844,
      "loss": 0.1347,
      "rewards/accuracies": 0.9524999856948853,
      "rewards/chosen": -1.5243346691131592,
      "rewards/margins": 4.38024377822876,
      "rewards/rejected": -5.904578685760498,
      "step": 4750
    },
    {
      "epoch": 1.8803996473699676,
      "grad_norm": 22.50926399230957,
      "learning_rate": 3.1721821985628946e-07,
      "logits/chosen": -1.7507199048995972,
      "logits/rejected": -1.587468147277832,
      "logps/chosen": -342.8749084472656,
      "logps/rejected": -330.3369140625,
      "loss": 0.154,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -1.4566929340362549,
      "rewards/margins": 4.056729793548584,
      "rewards/rejected": -5.513422012329102,
      "step": 4800
    },
    {
      "epoch": 1.8803996473699676,
      "eval_logits/chosen": -1.8217155933380127,
      "eval_logits/rejected": -1.761517882347107,
      "eval_logps/chosen": -376.3056335449219,
      "eval_logps/rejected": -332.8399658203125,
      "eval_loss": 0.528506338596344,
      "eval_rewards/accuracies": 0.7843464016914368,
      "eval_rewards/chosen": -2.3992199897766113,
      "eval_rewards/margins": 2.3415098190307617,
      "eval_rewards/rejected": -4.740729808807373,
      "eval_runtime": 182.1926,
      "eval_samples_per_second": 13.184,
      "eval_steps_per_second": 6.592,
      "step": 4800
    },
    {
      "epoch": 1.8999902047213242,
      "grad_norm": 41.3251953125,
      "learning_rate": 3.075234313313095e-07,
      "logits/chosen": -1.822758674621582,
      "logits/rejected": -1.6095324754714966,
      "logps/chosen": -368.2705078125,
      "logps/rejected": -363.7851867675781,
      "loss": 0.1114,
      "rewards/accuracies": 0.9599999785423279,
      "rewards/chosen": -1.1875016689300537,
      "rewards/margins": 4.392082691192627,
      "rewards/rejected": -5.579584121704102,
      "step": 4850
    },
    {
      "epoch": 1.9195807620726808,
      "grad_norm": 33.448936462402344,
      "learning_rate": 2.9791286713847106e-07,
      "logits/chosen": -1.874271035194397,
      "logits/rejected": -1.7505204677581787,
      "logps/chosen": -380.5478820800781,
      "logps/rejected": -355.3659362792969,
      "loss": 0.1139,
      "rewards/accuracies": 0.9549999833106995,
      "rewards/chosen": -1.4798643589019775,
      "rewards/margins": 4.445069789886475,
      "rewards/rejected": -5.924932956695557,
      "step": 4900
    },
    {
      "epoch": 1.9391713194240376,
      "grad_norm": 53.873172760009766,
      "learning_rate": 2.883907326901849e-07,
      "logits/chosen": -1.9296669960021973,
      "logits/rejected": -1.7788227796554565,
      "logps/chosen": -355.0895690917969,
      "logps/rejected": -336.3532409667969,
      "loss": 0.136,
      "rewards/accuracies": 0.9375,
      "rewards/chosen": -1.5472720861434937,
      "rewards/margins": 4.275333881378174,
      "rewards/rejected": -5.822606086730957,
      "step": 4950
    },
    {
      "epoch": 1.9587618767753943,
      "grad_norm": 66.68231964111328,
      "learning_rate": 2.7896119470357394e-07,
      "logits/chosen": -1.8380649089813232,
      "logits/rejected": -1.626741647720337,
      "logps/chosen": -367.21307373046875,
      "logps/rejected": -350.3079833984375,
      "loss": 0.1328,
      "rewards/accuracies": 0.9574999809265137,
      "rewards/chosen": -1.383242130279541,
      "rewards/margins": 4.56887674331665,
      "rewards/rejected": -5.95211935043335,
      "step": 5000
    },
    {
      "epoch": 1.9587618767753943,
      "eval_logits/chosen": -1.7905840873718262,
      "eval_logits/rejected": -1.730137586593628,
      "eval_logps/chosen": -377.50054931640625,
      "eval_logps/rejected": -334.88006591796875,
      "eval_loss": 0.5351826548576355,
      "eval_rewards/accuracies": 0.7847626805305481,
      "eval_rewards/chosen": -2.5187087059020996,
      "eval_rewards/margins": 2.4260356426239014,
      "eval_rewards/rejected": -4.94474458694458,
      "eval_runtime": 182.3765,
      "eval_samples_per_second": 13.171,
      "eval_steps_per_second": 6.585,
      "step": 5000
    },
    {
      "epoch": 1.9783524341267509,
      "grad_norm": 9.479029655456543,
      "learning_rate": 2.696283793771921e-07,
      "logits/chosen": -1.8894351720809937,
      "logits/rejected": -1.6450564861297607,
      "logps/chosen": -372.2572326660156,
      "logps/rejected": -358.4980773925781,
      "loss": 0.1521,
      "rewards/accuracies": 0.9325000047683716,
      "rewards/chosen": -1.5351108312606812,
      "rewards/margins": 4.4576897621154785,
      "rewards/rejected": -5.992800235748291,
      "step": 5050
    },
    {
      "epoch": 1.9979429914781075,
      "grad_norm": 84.90283203125,
      "learning_rate": 2.603963705854731e-07,
      "logits/chosen": -1.905731201171875,
      "logits/rejected": -1.6645513772964478,
      "logps/chosen": -361.50701904296875,
      "logps/rejected": -353.17059326171875,
      "loss": 0.1461,
      "rewards/accuracies": 0.9424999952316284,
      "rewards/chosen": -1.4440213441848755,
      "rewards/margins": 4.272027492523193,
      "rewards/rejected": -5.716048717498779,
      "step": 5100
    }
  ],
  "logging_steps": 50,
  "max_steps": 7659,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 300,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}