trainer_state.json

{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 3.0,
  "eval_steps": 500,
  "global_step": 1200,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "epoch": 0.06,
      "learning_rate": 1.0416666666666667e-07,
      "logps/chosen": -50.84611511230469,
      "logps/rejected": -68.16911315917969,
      "loss": 0.7128,
      "losses/dpo": 0.7152940630912781,
      "losses/sft": 1.772851586341858,
      "losses/total": 0.7152940630912781,
      "ref_logps/chosen": -50.76835632324219,
      "ref_logps/rejected": -68.15713500976562,
      "rewards/accuracies": 0.4920000433921814,
      "rewards/chosen": -0.007775493897497654,
      "rewards/margins": -0.006577868480235338,
      "rewards/rejected": -0.0011976259993389249,
      "step": 25
    },
    {
      "epoch": 0.12,
      "learning_rate": 2.0833333333333333e-07,
      "logps/chosen": -51.992218017578125,
      "logps/rejected": -67.8126220703125,
      "loss": 0.7059,
      "losses/dpo": 0.7131974697113037,
      "losses/sft": 1.781252384185791,
      "losses/total": 0.7131974697113037,
      "ref_logps/chosen": -52.02339172363281,
      "ref_logps/rejected": -67.7878646850586,
      "rewards/accuracies": 0.5110000371932983,
      "rewards/chosen": 0.003117597894743085,
      "rewards/margins": 0.005593638867139816,
      "rewards/rejected": -0.0024760416708886623,
      "step": 50
    },
    {
      "epoch": 0.19,
      "learning_rate": 3.1249999999999997e-07,
      "logps/chosen": -52.34601593017578,
      "logps/rejected": -67.67086029052734,
      "loss": 0.7097,
      "losses/dpo": 0.7076306939125061,
      "losses/sft": 1.7903097867965698,
      "losses/total": 0.7076306939125061,
      "ref_logps/chosen": -52.40769577026367,
      "ref_logps/rejected": -67.70091247558594,
      "rewards/accuracies": 0.49650001525878906,
      "rewards/chosen": 0.006168678868561983,
      "rewards/margins": 0.003163211978971958,
      "rewards/rejected": 0.003005466191098094,
      "step": 75
    },
    {
      "epoch": 0.25,
      "learning_rate": 4.1666666666666667e-07,
      "logps/chosen": -51.974483489990234,
      "logps/rejected": -68.44440460205078,
      "loss": 0.7115,
      "losses/dpo": 0.7194903492927551,
      "losses/sft": 1.777998924255371,
      "losses/total": 0.7194903492927551,
      "ref_logps/chosen": -51.95515441894531,
      "ref_logps/rejected": -68.4430923461914,
      "rewards/accuracies": 0.4794999957084656,
      "rewards/chosen": -0.0019327354384586215,
      "rewards/margins": -0.0018010810017585754,
      "rewards/rejected": -0.00013165354903321713,
      "step": 100
    },
    {
      "epoch": 0.31,
      "learning_rate": 4.976851851851852e-07,
      "logps/chosen": -51.21366882324219,
      "logps/rejected": -68.10318756103516,
      "loss": 0.7096,
      "losses/dpo": 0.7016645073890686,
      "losses/sft": 1.7784287929534912,
      "losses/total": 0.7016645073890686,
      "ref_logps/chosen": -51.175315856933594,
      "ref_logps/rejected": -68.03705596923828,
      "rewards/accuracies": 0.5074999928474426,
      "rewards/chosen": -0.003834722563624382,
      "rewards/margins": 0.002778256544843316,
      "rewards/rejected": -0.006612981669604778,
      "step": 125
    },
    {
      "epoch": 0.38,
      "learning_rate": 4.861111111111111e-07,
      "logps/chosen": -49.84953689575195,
      "logps/rejected": -66.59363555908203,
      "loss": 0.6989,
      "losses/dpo": 0.6881049275398254,
      "losses/sft": 1.7335822582244873,
      "losses/total": 0.6881049275398254,
      "ref_logps/chosen": -49.83342361450195,
      "ref_logps/rejected": -66.36032104492188,
      "rewards/accuracies": 0.5224999785423279,
      "rewards/chosen": -0.0016115426551550627,
      "rewards/margins": 0.021719880402088165,
      "rewards/rejected": -0.023331422358751297,
      "step": 150
    },
    {
      "epoch": 0.44,
      "learning_rate": 4.74537037037037e-07,
      "logps/chosen": -51.20610427856445,
      "logps/rejected": -68.37096405029297,
      "loss": 0.6965,
      "losses/dpo": 0.6989789605140686,
      "losses/sft": 1.8248052597045898,
      "losses/total": 0.6989789605140686,
      "ref_logps/chosen": -51.07723617553711,
      "ref_logps/rejected": -67.93907165527344,
      "rewards/accuracies": 0.5259999632835388,
      "rewards/chosen": -0.012886175885796547,
      "rewards/margins": 0.0303032249212265,
      "rewards/rejected": -0.0431894026696682,
      "step": 175
    },
    {
      "epoch": 0.5,
      "learning_rate": 4.6296296296296297e-07,
      "logps/chosen": -53.1021842956543,
      "logps/rejected": -69.64921569824219,
      "loss": 0.6821,
      "losses/dpo": 0.6905575394630432,
      "losses/sft": 1.7901018857955933,
      "losses/total": 0.6905575394630432,
      "ref_logps/chosen": -52.89280319213867,
      "ref_logps/rejected": -68.83776092529297,
      "rewards/accuracies": 0.5610000491142273,
      "rewards/chosen": -0.02093815989792347,
      "rewards/margins": 0.0602080263197422,
      "rewards/rejected": -0.08114618062973022,
      "step": 200
    },
    {
      "epoch": 0.56,
      "learning_rate": 4.513888888888889e-07,
      "logps/chosen": -51.60838317871094,
      "logps/rejected": -68.19721984863281,
      "loss": 0.6783,
      "losses/dpo": 0.6758668422698975,
      "losses/sft": 1.7487859725952148,
      "losses/total": 0.6758668422698975,
      "ref_logps/chosen": -51.219181060791016,
      "ref_logps/rejected": -67.0694580078125,
      "rewards/accuracies": 0.5529999732971191,
      "rewards/chosen": -0.03891964256763458,
      "rewards/margins": 0.0738559141755104,
      "rewards/rejected": -0.11277556419372559,
      "step": 225
    },
    {
      "epoch": 0.62,
      "learning_rate": 4.398148148148148e-07,
      "logps/chosen": -51.5633544921875,
      "logps/rejected": -70.04096221923828,
      "loss": 0.6606,
      "losses/dpo": 0.6762534976005554,
      "losses/sft": 1.7945468425750732,
      "losses/total": 0.6762534976005554,
      "ref_logps/chosen": -51.164485931396484,
      "ref_logps/rejected": -68.47702026367188,
      "rewards/accuracies": 0.5925000309944153,
      "rewards/chosen": -0.03988657891750336,
      "rewards/margins": 0.11650750786066055,
      "rewards/rejected": -0.1563940793275833,
      "step": 250
    },
    {
      "epoch": 0.69,
      "learning_rate": 4.2824074074074075e-07,
      "logps/chosen": -52.760066986083984,
      "logps/rejected": -71.27080535888672,
      "loss": 0.6541,
      "losses/dpo": 0.6559135317802429,
      "losses/sft": 1.8313995599746704,
      "losses/total": 0.6559135317802429,
      "ref_logps/chosen": -52.14683151245117,
      "ref_logps/rejected": -69.2181625366211,
      "rewards/accuracies": 0.6005000472068787,
      "rewards/chosen": -0.0613233856856823,
      "rewards/margins": 0.14394104480743408,
      "rewards/rejected": -0.20526443421840668,
      "step": 275
    },
    {
      "epoch": 0.75,
      "learning_rate": 4.1666666666666667e-07,
      "logps/chosen": -53.022369384765625,
      "logps/rejected": -70.22279357910156,
      "loss": 0.6444,
      "losses/dpo": 0.6332589387893677,
      "losses/sft": 1.8123092651367188,
      "losses/total": 0.6332589387893677,
      "ref_logps/chosen": -52.272945404052734,
      "ref_logps/rejected": -67.84944152832031,
      "rewards/accuracies": 0.6039999723434448,
      "rewards/chosen": -0.07494210451841354,
      "rewards/margins": 0.16239266097545624,
      "rewards/rejected": -0.2373347282409668,
      "step": 300
    },
    {
      "epoch": 0.81,
      "learning_rate": 4.050925925925926e-07,
      "logps/chosen": -52.53852462768555,
      "logps/rejected": -69.21473693847656,
      "loss": 0.6472,
      "losses/dpo": 0.6568192839622498,
      "losses/sft": 1.808381199836731,
      "losses/total": 0.6568192839622498,
      "ref_logps/chosen": -51.50153350830078,
      "ref_logps/rejected": -66.50575256347656,
      "rewards/accuracies": 0.6184999942779541,
      "rewards/chosen": -0.10369860380887985,
      "rewards/margins": 0.1671994924545288,
      "rewards/rejected": -0.27089810371398926,
      "step": 325
    },
    {
      "epoch": 0.88,
      "learning_rate": 3.9351851851851854e-07,
      "logps/chosen": -52.68033218383789,
      "logps/rejected": -69.9216079711914,
      "loss": 0.6322,
      "losses/dpo": 0.6414695978164673,
      "losses/sft": 1.7517987489700317,
      "losses/total": 0.6414695978164673,
      "ref_logps/chosen": -51.4389533996582,
      "ref_logps/rejected": -66.67315673828125,
      "rewards/accuracies": 0.6355000138282776,
      "rewards/chosen": -0.12413790076971054,
      "rewards/margins": 0.20070795714855194,
      "rewards/rejected": -0.3248458802700043,
      "step": 350
    },
    {
      "epoch": 0.94,
      "learning_rate": 3.819444444444444e-07,
      "logps/chosen": -53.01972579956055,
      "logps/rejected": -69.83106231689453,
      "loss": 0.6254,
      "losses/dpo": 0.6416024565696716,
      "losses/sft": 1.806755781173706,
      "losses/total": 0.6416024565696716,
      "ref_logps/chosen": -51.760948181152344,
      "ref_logps/rejected": -66.29093170166016,
      "rewards/accuracies": 0.6260000467300415,
      "rewards/chosen": -0.12587812542915344,
      "rewards/margins": 0.22813528776168823,
      "rewards/rejected": -0.3540134131908417,
      "step": 375
    },
    {
      "epoch": 1.0,
      "learning_rate": 3.703703703703703e-07,
      "logps/chosen": -53.866825103759766,
      "logps/rejected": -73.0006332397461,
      "loss": 0.6223,
      "losses/dpo": 0.6281777024269104,
      "losses/sft": 1.8359609842300415,
      "losses/total": 0.6281777024269104,
      "ref_logps/chosen": -52.29502868652344,
      "ref_logps/rejected": -68.76205444335938,
      "rewards/accuracies": 0.624500036239624,
      "rewards/chosen": -0.15717901289463043,
      "rewards/margins": 0.26667752861976624,
      "rewards/rejected": -0.42385655641555786,
      "step": 400
    },
    {
      "epoch": 1.06,
      "learning_rate": 3.587962962962963e-07,
      "logps/chosen": -52.83345031738281,
      "logps/rejected": -73.19969940185547,
      "loss": 0.6044,
      "losses/dpo": 0.6066713929176331,
      "losses/sft": 1.8482356071472168,
      "losses/total": 0.6066713929176331,
      "ref_logps/chosen": -51.23924255371094,
      "ref_logps/rejected": -68.46554565429688,
      "rewards/accuracies": 0.658500075340271,
      "rewards/chosen": -0.15942102670669556,
      "rewards/margins": 0.31399470567703247,
      "rewards/rejected": -0.4734157621860504,
      "step": 425
    },
    {
      "epoch": 1.12,
      "learning_rate": 3.472222222222222e-07,
      "logps/chosen": -53.00102996826172,
      "logps/rejected": -71.68830871582031,
      "loss": 0.6107,
      "losses/dpo": 0.5940313935279846,
      "losses/sft": 1.8907349109649658,
      "losses/total": 0.5940313935279846,
      "ref_logps/chosen": -51.17875671386719,
      "ref_logps/rejected": -66.69224548339844,
      "rewards/accuracies": 0.6355000138282776,
      "rewards/chosen": -0.18222743272781372,
      "rewards/margins": 0.3173789978027344,
      "rewards/rejected": -0.4996064305305481,
      "step": 450
    },
    {
      "epoch": 1.19,
      "learning_rate": 3.3564814814814815e-07,
      "logps/chosen": -54.13747024536133,
      "logps/rejected": -75.39285278320312,
      "loss": 0.5906,
      "losses/dpo": 0.6001964807510376,
      "losses/sft": 1.837428331375122,
      "losses/total": 0.6001964807510376,
      "ref_logps/chosen": -52.013328552246094,
      "ref_logps/rejected": -69.39274597167969,
      "rewards/accuracies": 0.6559999585151672,
      "rewards/chosen": -0.21241408586502075,
      "rewards/margins": 0.387596994638443,
      "rewards/rejected": -0.6000111103057861,
      "step": 475
    },
    {
      "epoch": 1.25,
      "learning_rate": 3.2407407407407406e-07,
      "logps/chosen": -52.84211730957031,
      "logps/rejected": -73.44519805908203,
      "loss": 0.588,
      "losses/dpo": 0.5720361471176147,
      "losses/sft": 1.82402765750885,
      "losses/total": 0.5720361471176147,
      "ref_logps/chosen": -50.860145568847656,
      "ref_logps/rejected": -67.45731353759766,
      "rewards/accuracies": 0.6580000519752502,
      "rewards/chosen": -0.1981971114873886,
      "rewards/margins": 0.40059128403663635,
      "rewards/rejected": -0.5987883806228638,
      "step": 500
    },
    {
      "epoch": 1.31,
      "learning_rate": 3.1249999999999997e-07,
      "logps/chosen": -54.73566436767578,
      "logps/rejected": -73.35672760009766,
      "loss": 0.5864,
      "losses/dpo": 0.5692346096038818,
      "losses/sft": 1.8467037677764893,
      "losses/total": 0.5692346096038818,
      "ref_logps/chosen": -52.32421875,
      "ref_logps/rejected": -66.84322357177734,
      "rewards/accuracies": 0.6690000295639038,
      "rewards/chosen": -0.24114486575126648,
      "rewards/margins": 0.4102060794830322,
      "rewards/rejected": -0.6513509154319763,
      "step": 525
    },
    {
      "epoch": 1.38,
      "learning_rate": 3.0092592592592594e-07,
      "logps/chosen": -54.00200653076172,
      "logps/rejected": -77.5279312133789,
      "loss": 0.5712,
      "losses/dpo": 0.5749830007553101,
      "losses/sft": 1.8555569648742676,
      "losses/total": 0.5749830007553101,
      "ref_logps/chosen": -51.46855163574219,
      "ref_logps/rejected": -70.09744262695312,
      "rewards/accuracies": 0.6720000505447388,
      "rewards/chosen": -0.25334587693214417,
      "rewards/margins": 0.48970410227775574,
      "rewards/rejected": -0.7430499196052551,
      "step": 550
    },
    {
      "epoch": 1.44,
      "learning_rate": 2.8935185185185185e-07,
      "logps/chosen": -53.216957092285156,
      "logps/rejected": -74.7680892944336,
      "loss": 0.5712,
      "losses/dpo": 0.5629280209541321,
      "losses/sft": 1.8214166164398193,
      "losses/total": 0.5629280209541321,
      "ref_logps/chosen": -50.563720703125,
      "ref_logps/rejected": -67.18008422851562,
      "rewards/accuracies": 0.6755000352859497,
      "rewards/chosen": -0.2653237581253052,
      "rewards/margins": 0.4934755265712738,
      "rewards/rejected": -0.7587993144989014,
      "step": 575
    },
    {
      "epoch": 1.5,
      "learning_rate": 2.7777777777777776e-07,
      "logps/chosen": -55.059600830078125,
      "logps/rejected": -76.23825073242188,
      "loss": 0.5577,
      "losses/dpo": 0.5477968454360962,
      "losses/sft": 1.9264967441558838,
      "losses/total": 0.5477968454360962,
      "ref_logps/chosen": -52.09434509277344,
      "ref_logps/rejected": -67.781005859375,
      "rewards/accuracies": 0.6880000233650208,
      "rewards/chosen": -0.2965255081653595,
      "rewards/margins": 0.5491979122161865,
      "rewards/rejected": -0.8457233905792236,
      "step": 600
    },
    {
      "epoch": 1.56,
      "learning_rate": 2.662037037037037e-07,
      "logps/chosen": -56.8978271484375,
      "logps/rejected": -78.7895736694336,
      "loss": 0.5667,
      "losses/dpo": 0.5594893097877502,
      "losses/sft": 1.9116603136062622,
      "losses/total": 0.5594893097877502,
      "ref_logps/chosen": -53.51485824584961,
      "ref_logps/rejected": -69.69821166992188,
      "rewards/accuracies": 0.6595000624656677,
      "rewards/chosen": -0.338296502828598,
      "rewards/margins": 0.5708397030830383,
      "rewards/rejected": -0.9091362953186035,
      "step": 625
    },
    {
      "epoch": 1.62,
      "learning_rate": 2.5462962962962963e-07,
      "logps/chosen": -55.704219818115234,
      "logps/rejected": -77.06362915039062,
      "loss": 0.5734,
      "losses/dpo": 0.583986222743988,
      "losses/sft": 1.8731328248977661,
      "losses/total": 0.583986222743988,
      "ref_logps/chosen": -52.17438888549805,
      "ref_logps/rejected": -67.96855926513672,
      "rewards/accuracies": 0.6725000143051147,
      "rewards/chosen": -0.3529825508594513,
      "rewards/margins": 0.5565236806869507,
      "rewards/rejected": -0.9095062017440796,
      "step": 650
    },
    {
      "epoch": 1.69,
      "learning_rate": 2.4305555555555555e-07,
      "logps/chosen": -54.6390266418457,
      "logps/rejected": -76.69291687011719,
      "loss": 0.5568,
      "losses/dpo": 0.5715891718864441,
      "losses/sft": 1.8541003465652466,
      "losses/total": 0.5715891718864441,
      "ref_logps/chosen": -51.134674072265625,
      "ref_logps/rejected": -67.13263702392578,
      "rewards/accuracies": 0.687000036239624,
      "rewards/chosen": -0.3504358232021332,
      "rewards/margins": 0.605591893196106,
      "rewards/rejected": -0.9560276865959167,
      "step": 675
    },
    {
      "epoch": 1.75,
      "learning_rate": 2.3148148148148148e-07,
      "logps/chosen": -54.50948715209961,
      "logps/rejected": -77.46994018554688,
      "loss": 0.5657,
      "losses/dpo": 0.5799014568328857,
      "losses/sft": 1.8863662481307983,
      "losses/total": 0.5799014568328857,
      "ref_logps/chosen": -50.88352966308594,
      "ref_logps/rejected": -67.8268051147461,
      "rewards/accuracies": 0.6755000352859497,
      "rewards/chosen": -0.36259526014328003,
      "rewards/margins": 0.6017180681228638,
      "rewards/rejected": -0.9643133282661438,
      "step": 700
    },
    {
      "epoch": 1.81,
      "learning_rate": 2.199074074074074e-07,
      "logps/chosen": -55.129295349121094,
      "logps/rejected": -77.04840087890625,
      "loss": 0.5636,
      "losses/dpo": 0.5747159123420715,
      "losses/sft": 1.926343560218811,
      "losses/total": 0.5747159123420715,
      "ref_logps/chosen": -51.24678039550781,
      "ref_logps/rejected": -67.05614471435547,
      "rewards/accuracies": 0.6730000376701355,
      "rewards/chosen": -0.3882514238357544,
      "rewards/margins": 0.6109741926193237,
      "rewards/rejected": -0.9992256760597229,
      "step": 725
    },
    {
      "epoch": 1.88,
      "learning_rate": 2.0833333333333333e-07,
      "logps/chosen": -55.22993087768555,
      "logps/rejected": -78.04380798339844,
      "loss": 0.5575,
      "losses/dpo": 0.5746473073959351,
      "losses/sft": 1.8473073244094849,
      "losses/total": 0.5746473073959351,
      "ref_logps/chosen": -51.49522399902344,
      "ref_logps/rejected": -67.86705017089844,
      "rewards/accuracies": 0.6645000576972961,
      "rewards/chosen": -0.3734706938266754,
      "rewards/margins": 0.6442059278488159,
      "rewards/rejected": -1.0176767110824585,
      "step": 750
    },
    {
      "epoch": 1.94,
      "learning_rate": 1.9675925925925927e-07,
      "logps/chosen": -54.95744323730469,
      "logps/rejected": -77.55332946777344,
      "loss": 0.5428,
      "losses/dpo": 0.5475051999092102,
      "losses/sft": 1.8516862392425537,
      "losses/total": 0.5475051999092102,
      "ref_logps/chosen": -51.13042449951172,
      "ref_logps/rejected": -66.90882110595703,
      "rewards/accuracies": 0.6959999799728394,
      "rewards/chosen": -0.382702112197876,
      "rewards/margins": 0.6817485690116882,
      "rewards/rejected": -1.064450740814209,
      "step": 775
    },
    {
      "epoch": 2.0,
      "learning_rate": 1.8518518518518516e-07,
      "logps/chosen": -56.311126708984375,
      "logps/rejected": -79.54733276367188,
      "loss": 0.5453,
      "losses/dpo": 0.529507577419281,
      "losses/sft": 1.9264029264450073,
      "losses/total": 0.529507577419281,
      "ref_logps/chosen": -51.99349594116211,
      "ref_logps/rejected": -68.2928237915039,
      "rewards/accuracies": 0.7039999961853027,
      "rewards/chosen": -0.4317638874053955,
      "rewards/margins": 0.6936854720115662,
      "rewards/rejected": -1.1254491806030273,
      "step": 800
    },
    {
      "epoch": 2.06,
      "learning_rate": 1.736111111111111e-07,
      "logps/chosen": -55.88136672973633,
      "logps/rejected": -78.89193725585938,
      "loss": 0.5536,
      "losses/dpo": 0.5027932524681091,
      "losses/sft": 1.888191819190979,
      "losses/total": 0.5027932524681091,
      "ref_logps/chosen": -51.50459289550781,
      "ref_logps/rejected": -67.58809661865234,
      "rewards/accuracies": 0.7039999961853027,
      "rewards/chosen": -0.43767789006233215,
      "rewards/margins": 0.6927057504653931,
      "rewards/rejected": -1.1303836107254028,
      "step": 825
    },
    {
      "epoch": 2.12,
      "learning_rate": 1.6203703703703703e-07,
      "logps/chosen": -56.851905822753906,
      "logps/rejected": -79.84717559814453,
      "loss": 0.5519,
      "losses/dpo": 0.5627533197402954,
      "losses/sft": 1.950637936592102,
      "losses/total": 0.5627533197402954,
      "ref_logps/chosen": -52.47712326049805,
      "ref_logps/rejected": -68.6231918334961,
      "rewards/accuracies": 0.6880000233650208,
      "rewards/chosen": -0.4374789893627167,
      "rewards/margins": 0.6849193572998047,
      "rewards/rejected": -1.1223982572555542,
      "step": 850
    },
    {
      "epoch": 2.19,
      "learning_rate": 1.5046296296296297e-07,
      "logps/chosen": -56.792240142822266,
      "logps/rejected": -80.210205078125,
      "loss": 0.5433,
      "losses/dpo": 0.562707245349884,
      "losses/sft": 1.9036242961883545,
      "losses/total": 0.562707245349884,
      "ref_logps/chosen": -52.128883361816406,
      "ref_logps/rejected": -68.35295104980469,
      "rewards/accuracies": 0.6974999904632568,
      "rewards/chosen": -0.46633559465408325,
      "rewards/margins": 0.719389796257019,
      "rewards/rejected": -1.185725450515747,
      "step": 875
    },
    {
      "epoch": 2.25,
      "learning_rate": 1.3888888888888888e-07,
      "logps/chosen": -56.366416931152344,
      "logps/rejected": -81.07738494873047,
      "loss": 0.5341,
      "losses/dpo": 0.5471935868263245,
      "losses/sft": 1.9060190916061401,
      "losses/total": 0.5471935868263245,
      "ref_logps/chosen": -51.77671432495117,
      "ref_logps/rejected": -68.86495971679688,
      "rewards/accuracies": 0.6914999485015869,
      "rewards/chosen": -0.45896950364112854,
      "rewards/margins": 0.7622722387313843,
      "rewards/rejected": -1.2212417125701904,
      "step": 900
    },
    {
      "epoch": 2.31,
      "learning_rate": 1.2731481481481482e-07,
      "logps/chosen": -56.640018463134766,
      "logps/rejected": -81.28205871582031,
      "loss": 0.5195,
      "losses/dpo": 0.5150865316390991,
      "losses/sft": 1.918342113494873,
      "losses/total": 0.5150865316390991,
      "ref_logps/chosen": -52.07315444946289,
      "ref_logps/rejected": -68.57891082763672,
      "rewards/accuracies": 0.7275000214576721,
      "rewards/chosen": -0.4566872715950012,
      "rewards/margins": 0.8136279582977295,
      "rewards/rejected": -1.2703152894973755,
      "step": 925
    },
    {
      "epoch": 2.38,
      "learning_rate": 1.1574074074074074e-07,
      "logps/chosen": -56.610965728759766,
      "logps/rejected": -82.6984634399414,
      "loss": 0.5267,
      "losses/dpo": 0.5297408103942871,
      "losses/sft": 1.9613755941390991,
      "losses/total": 0.5297408103942871,
      "ref_logps/chosen": -51.79991149902344,
      "ref_logps/rejected": -69.79439544677734,
      "rewards/accuracies": 0.7074999809265137,
      "rewards/chosen": -0.4811055660247803,
      "rewards/margins": 0.8093020915985107,
      "rewards/rejected": -1.2904075384140015,
      "step": 950
    },
    {
      "epoch": 2.44,
      "learning_rate": 1.0416666666666667e-07,
      "logps/chosen": -55.42496109008789,
      "logps/rejected": -80.11758422851562,
      "loss": 0.5346,
      "losses/dpo": 0.5188722610473633,
      "losses/sft": 1.9635555744171143,
      "losses/total": 0.5188722610473633,
      "ref_logps/chosen": -50.690673828125,
      "ref_logps/rejected": -67.66686248779297,
      "rewards/accuracies": 0.7055000066757202,
      "rewards/chosen": -0.47342926263809204,
      "rewards/margins": 0.7716432213783264,
      "rewards/rejected": -1.2450724840164185,
      "step": 975
    },
    {
      "epoch": 2.5,
      "learning_rate": 9.259259259259258e-08,
      "logps/chosen": -55.2717399597168,
      "logps/rejected": -78.23765563964844,
      "loss": 0.5358,
      "losses/dpo": 0.522473156452179,
      "losses/sft": 1.9380919933319092,
      "losses/total": 0.522473156452179,
      "ref_logps/chosen": -50.576473236083984,
      "ref_logps/rejected": -65.8857192993164,
      "rewards/accuracies": 0.6880000233650208,
      "rewards/chosen": -0.46952569484710693,
      "rewards/margins": 0.7656675577163696,
      "rewards/rejected": -1.2351933717727661,
      "step": 1000
    },
    {
      "epoch": 2.56,
      "learning_rate": 8.101851851851852e-08,
      "logps/chosen": -57.238101959228516,
      "logps/rejected": -80.15925598144531,
      "loss": 0.5477,
      "losses/dpo": 0.5454632043838501,
      "losses/sft": 1.9586001634597778,
      "losses/total": 0.5454632043838501,
      "ref_logps/chosen": -51.92245101928711,
      "ref_logps/rejected": -67.11357879638672,
      "rewards/accuracies": 0.6919999718666077,
      "rewards/chosen": -0.5315641164779663,
      "rewards/margins": 0.7730042934417725,
      "rewards/rejected": -1.3045685291290283,
      "step": 1025
    },
    {
      "epoch": 2.62,
      "learning_rate": 6.944444444444444e-08,
      "logps/chosen": -56.340274810791016,
      "logps/rejected": -78.74750518798828,
      "loss": 0.5353,
      "losses/dpo": 0.564401388168335,
      "losses/sft": 1.9238648414611816,
      "losses/total": 0.564401388168335,
      "ref_logps/chosen": -51.298805236816406,
      "ref_logps/rejected": -66.08538818359375,
      "rewards/accuracies": 0.7080000042915344,
      "rewards/chosen": -0.5041469931602478,
      "rewards/margins": 0.7620646953582764,
      "rewards/rejected": -1.2662116289138794,
      "step": 1050
    },
    {
      "epoch": 2.69,
      "learning_rate": 5.787037037037037e-08,
      "logps/chosen": -58.54380416870117,
      "logps/rejected": -82.85121154785156,
      "loss": 0.5379,
      "losses/dpo": 0.4951947033405304,
      "losses/sft": 1.927669882774353,
      "losses/total": 0.4951947033405304,
      "ref_logps/chosen": -53.125526428222656,
      "ref_logps/rejected": -69.30874633789062,
      "rewards/accuracies": 0.699999988079071,
      "rewards/chosen": -0.5418279767036438,
      "rewards/margins": 0.8124186992645264,
      "rewards/rejected": -1.3542466163635254,
      "step": 1075
    },
    {
      "epoch": 2.75,
      "learning_rate": 4.629629629629629e-08,
      "logps/chosen": -56.923858642578125,
      "logps/rejected": -81.64384460449219,
      "loss": 0.5352,
      "losses/dpo": 0.5503857135772705,
      "losses/sft": 1.9750890731811523,
      "losses/total": 0.5503857135772705,
      "ref_logps/chosen": -51.61204528808594,
      "ref_logps/rejected": -68.07569122314453,
      "rewards/accuracies": 0.7145000696182251,
      "rewards/chosen": -0.5311816930770874,
      "rewards/margins": 0.8256343603134155,
      "rewards/rejected": -1.356816053390503,
      "step": 1100
    },
    {
      "epoch": 2.81,
      "learning_rate": 3.472222222222222e-08,
      "logps/chosen": -57.147186279296875,
      "logps/rejected": -79.592529296875,
      "loss": 0.541,
      "losses/dpo": 0.5187271237373352,
      "losses/sft": 1.9453308582305908,
      "losses/total": 0.5187271237373352,
      "ref_logps/chosen": -52.0300178527832,
      "ref_logps/rejected": -66.7751693725586,
      "rewards/accuracies": 0.7010000348091125,
      "rewards/chosen": -0.5117166042327881,
      "rewards/margins": 0.7700192332267761,
      "rewards/rejected": -1.281735897064209,
      "step": 1125
    },
    {
      "epoch": 2.88,
      "learning_rate": 2.3148148148148144e-08,
      "logps/chosen": -56.04924011230469,
      "logps/rejected": -82.78221893310547,
      "loss": 0.5138,
      "losses/dpo": 0.5014563202857971,
      "losses/sft": 1.892021656036377,
      "losses/total": 0.5014563202857971,
      "ref_logps/chosen": -50.947792053222656,
      "ref_logps/rejected": -68.86062622070312,
      "rewards/accuracies": 0.7239999175071716,
      "rewards/chosen": -0.5101450085639954,
      "rewards/margins": 0.8820143938064575,
      "rewards/rejected": -1.3921594619750977,
      "step": 1150
    },
    {
      "epoch": 2.94,
      "learning_rate": 1.1574074074074072e-08,
      "logps/chosen": -57.95354461669922,
      "logps/rejected": -81.48481750488281,
      "loss": 0.5322,
      "losses/dpo": 0.5276296138763428,
      "losses/sft": 1.914094090461731,
      "losses/total": 0.5276296138763428,
      "ref_logps/chosen": -52.49882507324219,
      "ref_logps/rejected": -68.10453796386719,
      "rewards/accuracies": 0.7064999938011169,
      "rewards/chosen": -0.5454717874526978,
      "rewards/margins": 0.7925567030906677,
      "rewards/rejected": -1.3380285501480103,
      "step": 1175
    },
    {
      "epoch": 3.0,
      "learning_rate": 0.0,
      "logps/chosen": -58.058658599853516,
      "logps/rejected": -80.37024688720703,
      "loss": 0.5418,
      "losses/dpo": 0.5331112146377563,
      "losses/sft": 1.9555615186691284,
      "losses/total": 0.5331112146377563,
      "ref_logps/chosen": -52.748592376708984,
      "ref_logps/rejected": -67.33932495117188,
      "rewards/accuracies": 0.6899999976158142,
      "rewards/chosen": -0.5310066342353821,
      "rewards/margins": 0.7720865607261658,
      "rewards/rejected": -1.3030930757522583,
      "step": 1200
    },
    {
      "epoch": 3.0,
      "step": 1200,
      "total_flos": 0.0,
      "train_loss": 0.5943309187889099,
      "train_runtime": 16038.0463,
      "train_samples_per_second": 5.984,
      "train_steps_per_second": 0.075
    }
  ],
  "logging_steps": 25,
  "max_steps": 1200,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 3,
  "save_steps": 500,
  "stateful_callbacks": {},
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}