{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 2.0,
  "eval_steps": 500,
  "global_step": 10000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 65.065,
      "epoch": 0.02,
      "grad_norm": 10.375,
      "kl": 0.0006580278992169042,
      "learning_rate": 5.000000000000001e-07,
      "loss": -0.0,
      "match_ratio": 0.995,
      "reward": 0.8990143708884716,
      "reward_std": 0.46338544798083603,
      "rewards/reward_func": 0.8990143708884716,
      "step": 100
    },
    {
      "completion_length": 65.4625,
      "epoch": 0.04,
      "grad_norm": 6.875,
      "kl": 0.0006705577400316542,
      "learning_rate": 1.0000000000000002e-06,
      "loss": -0.0,
      "match_ratio": 1.0,
      "reward": 0.7165287194028497,
      "reward_std": 0.40350831425283107,
      "rewards/reward_func": 0.7165287194028497,
      "step": 200
    },
    {
      "completion_length": 61.175,
      "epoch": 0.06,
      "grad_norm": 8.4375,
      "kl": 0.0007126682825037278,
      "learning_rate": 1.5e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7286543997749686,
      "reward_std": 0.4331769395247102,
      "rewards/reward_func": 0.7286543997749686,
      "step": 300
    },
    {
      "completion_length": 54.655,
      "epoch": 0.08,
      "grad_norm": 11.9375,
      "kl": 0.0008965998092025984,
      "learning_rate": 2.0000000000000003e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7439908282458783,
      "reward_std": 0.4567913323547691,
      "rewards/reward_func": 0.7439908282458783,
      "step": 400
    },
    {
      "completion_length": 62.1025,
      "epoch": 0.1,
      "grad_norm": 15.8125,
      "kl": 0.0019434646295849235,
      "learning_rate": 2.5e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8668537394329906,
      "reward_std": 0.3836937860772014,
      "rewards/reward_func": 0.8668537394329906,
      "step": 500
    },
    {
      "completion_length": 58.14,
      "epoch": 0.12,
      "grad_norm": 20.5,
      "kl": 0.0041620647069066765,
      "learning_rate": 3e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7577041421830654,
      "reward_std": 0.43559244139119985,
      "rewards/reward_func": 0.7577041421830654,
      "step": 600
    },
    {
      "completion_length": 60.0,
      "epoch": 0.14,
      "grad_norm": 10.6875,
      "kl": 0.008898616410442628,
      "learning_rate": 3.5e-06,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.8700515530258417,
      "reward_std": 0.45400316243059935,
      "rewards/reward_func": 0.8700515530258417,
      "step": 700
    },
    {
      "completion_length": 58.1275,
      "epoch": 0.16,
      "grad_norm": 7.375,
      "kl": 0.0189549465168966,
      "learning_rate": 4.000000000000001e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.7298830785602332,
      "reward_std": 0.4231558512337506,
      "rewards/reward_func": 0.7298830785602332,
      "step": 800
    },
    {
      "completion_length": 55.55,
      "epoch": 0.18,
      "grad_norm": 13.3125,
      "kl": 0.04016699714120477,
      "learning_rate": 4.5e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8830712201073766,
      "reward_std": 0.3306662117503583,
      "rewards/reward_func": 0.8830712201073766,
      "step": 900
    },
    {
      "completion_length": 55.0225,
      "epoch": 0.2,
      "grad_norm": 10.5625,
      "kl": 0.06088939258828759,
      "learning_rate": 5e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7573206969350577,
      "reward_std": 0.32937701970338823,
      "rewards/reward_func": 0.7573206969350577,
      "step": 1000
    },
    {
      "completion_length": 59.6125,
      "epoch": 0.22,
      "grad_norm": 11.0625,
      "kl": 0.03887372653000057,
      "learning_rate": 4.99847706754774e-06,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.8389806092530488,
      "reward_std": 0.3803154364787042,
      "rewards/reward_func": 0.8389806092530488,
      "step": 1100
    },
    {
      "completion_length": 62.01,
      "epoch": 0.24,
      "grad_norm": 8.75,
      "kl": 0.9295957709802315,
      "learning_rate": 4.993910125649561e-06,
      "loss": 0.0001,
      "match_ratio": 0.9925,
      "reward": 0.8083837843686342,
      "reward_std": 0.4002057794481516,
      "rewards/reward_func": 0.8083837843686342,
      "step": 1200
    },
    {
      "completion_length": 61.6575,
      "epoch": 0.26,
      "grad_norm": 12.8125,
      "kl": 0.5295558683061973,
      "learning_rate": 4.986304738420684e-06,
      "loss": 0.0001,
      "match_ratio": 0.9925,
      "reward": 0.8700573812425136,
      "reward_std": 0.41881847178563475,
      "rewards/reward_func": 0.8700573812425136,
      "step": 1300
    },
    {
      "completion_length": 55.82,
      "epoch": 0.28,
      "grad_norm": 21.0,
      "kl": 0.2268725570756942,
      "learning_rate": 4.975670171853926e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7677264379709959,
      "reward_std": 0.4127253815624863,
      "rewards/reward_func": 0.7677264379709959,
      "step": 1400
    },
    {
      "completion_length": 61.605,
      "epoch": 0.3,
      "grad_norm": 15.25,
      "kl": 2.662307023219764,
      "learning_rate": 4.962019382530521e-06,
      "loss": 0.0003,
      "match_ratio": 0.9975,
      "reward": 0.8300903634727002,
      "reward_std": 0.330243071205914,
      "rewards/reward_func": 0.8300903634727002,
      "step": 1500
    },
    {
      "completion_length": 57.8975,
      "epoch": 0.32,
      "grad_norm": 24.25,
      "kl": 0.9898469369392842,
      "learning_rate": 4.9453690018345144e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8029376929998397,
      "reward_std": 0.3678751669218764,
      "rewards/reward_func": 0.8029376929998397,
      "step": 1600
    },
    {
      "completion_length": 52.7,
      "epoch": 0.34,
      "grad_norm": 3.453125,
      "kl": 116.13146778639405,
      "learning_rate": 4.925739315689991e-06,
      "loss": 0.0116,
      "match_ratio": 1.0,
      "reward": 0.8077524190768599,
      "reward_std": 0.33210189862176775,
      "rewards/reward_func": 0.8077524190768599,
      "step": 1700
    },
    {
      "completion_length": 54.5,
      "epoch": 0.36,
      "grad_norm": 12.25,
      "kl": 5.921828000650276,
      "learning_rate": 4.903154239845798e-06,
      "loss": 0.0006,
      "match_ratio": 1.0,
      "reward": 0.8151757456362247,
      "reward_std": 0.3074088580603711,
      "rewards/reward_func": 0.8151757456362247,
      "step": 1800
    },
    {
      "completion_length": 59.43,
      "epoch": 0.38,
      "grad_norm": 22.25,
      "kl": 0.22536800906993448,
      "learning_rate": 4.8776412907378845e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8543619333952666,
      "reward_std": 0.3683507715538144,
      "rewards/reward_func": 0.8543619333952666,
      "step": 1900
    },
    {
      "completion_length": 59.705,
      "epoch": 0.4,
      "grad_norm": 5.21875,
      "kl": 0.2842459925811272,
      "learning_rate": 4.849231551964771e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.9203101838380099,
      "reward_std": 0.3454422113858163,
      "rewards/reward_func": 0.9203101838380099,
      "step": 2000
    },
    {
      "completion_length": 65.095,
      "epoch": 0.42,
      "grad_norm": 9.125,
      "kl": 0.16623427679762245,
      "learning_rate": 4.817959636416969e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.9261023019999266,
      "reward_std": 0.3223581924289465,
      "rewards/reward_func": 0.9261023019999266,
      "step": 2100
    },
    {
      "completion_length": 62.8125,
      "epoch": 0.44,
      "grad_norm": 11.0625,
      "kl": 0.10455320389475674,
      "learning_rate": 4.783863644106502e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.9122521196585148,
      "reward_std": 0.37498483614996075,
      "rewards/reward_func": 0.9122521196585148,
      "step": 2200
    },
    {
      "completion_length": 71.1275,
      "epoch": 0.46,
      "grad_norm": 12.5,
      "kl": 0.5584425710327924,
      "learning_rate": 4.746985115747918e-06,
      "loss": 0.0001,
      "match_ratio": 0.9925,
      "reward": 0.792227897644043,
      "reward_std": 0.4075765323080123,
      "rewards/reward_func": 0.792227897644043,
      "step": 2300
    },
    {
      "completion_length": 68.27,
      "epoch": 0.48,
      "grad_norm": 10.625,
      "kl": 1.3814555319957436,
      "learning_rate": 4.707368982147318e-06,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.8301830168347806,
      "reward_std": 0.36762866189703347,
      "rewards/reward_func": 0.8301830168347806,
      "step": 2400
    },
    {
      "completion_length": 66.965,
      "epoch": 0.5,
      "grad_norm": 12.6875,
      "kl": 0.8518368338712026,
      "learning_rate": 4.665063509461098e-06,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.8910126995295287,
      "reward_std": 0.3965667562186718,
      "rewards/reward_func": 0.8910126995295287,
      "step": 2500
    },
    {
      "completion_length": 70.73,
      "epoch": 0.52,
      "grad_norm": 17.75,
      "kl": 0.5207290647923947,
      "learning_rate": 4.620120240391065e-06,
      "loss": 0.0001,
      "match_ratio": 0.9925,
      "reward": 0.8480577088147402,
      "reward_std": 0.4258584909327328,
      "rewards/reward_func": 0.8480577088147402,
      "step": 2600
    },
    {
      "completion_length": 59.6575,
      "epoch": 0.54,
      "grad_norm": 11.375,
      "kl": 0.7336286423553247,
      "learning_rate": 4.572593931387604e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.928214335795492,
      "reward_std": 0.37614597208797934,
      "rewards/reward_func": 0.928214335795492,
      "step": 2700
    },
    {
      "completion_length": 63.8775,
      "epoch": 0.56,
      "grad_norm": 9.5625,
      "kl": 0.33108121431432663,
      "learning_rate": 4.522542485937369e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.7988891634345054,
      "reward_std": 0.36210680682212115,
      "rewards/reward_func": 0.7988891634345054,
      "step": 2800
    },
    {
      "completion_length": 59.67,
      "epoch": 0.58,
      "grad_norm": 8.75,
      "kl": 72.35348623547704,
      "learning_rate": 4.470026884016805e-06,
      "loss": 0.0072,
      "match_ratio": 0.9975,
      "reward": 0.8135686150938273,
      "reward_std": 0.3942835557647049,
      "rewards/reward_func": 0.8135686150938273,
      "step": 2900
    },
    {
      "completion_length": 56.305,
      "epoch": 0.6,
      "grad_norm": 10.0,
      "kl": 0.14451225536875426,
      "learning_rate": 4.415111107797445e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8231964718922973,
      "reward_std": 0.3244973301887512,
      "rewards/reward_func": 0.8231964718922973,
      "step": 3000
    },
    {
      "completion_length": 59.705,
      "epoch": 0.62,
      "grad_norm": 10.0,
      "kl": 0.16642916494980453,
      "learning_rate": 4.357862063693486e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8778230049461127,
      "reward_std": 0.36886056323535743,
      "rewards/reward_func": 0.8778230049461127,
      "step": 3100
    },
    {
      "completion_length": 56.71,
      "epoch": 0.64,
      "grad_norm": 8.875,
      "kl": 0.18644527865573765,
      "learning_rate": 4.2983495008466285e-06,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.7874290134198964,
      "reward_std": 0.38642477702349426,
      "rewards/reward_func": 0.7874290134198964,
      "step": 3200
    },
    {
      "completion_length": 56.68,
      "epoch": 0.66,
      "grad_norm": 27.0,
      "kl": 0.22375864623580127,
      "learning_rate": 4.236645926147493e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8582109183818102,
      "reward_std": 0.36429890371393414,
      "rewards/reward_func": 0.8582109183818102,
      "step": 3300
    },
    {
      "completion_length": 58.835,
      "epoch": 0.68,
      "grad_norm": 6.125,
      "kl": 0.22189826945774258,
      "learning_rate": 4.172826515897146e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7524572538957,
      "reward_std": 0.35272345967590807,
      "rewards/reward_func": 0.7524572538957,
      "step": 3400
    },
    {
      "completion_length": 50.8375,
      "epoch": 0.7,
      "grad_norm": 14.375,
      "kl": 0.9133326725219376,
      "learning_rate": 4.106969024216348e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8736884651333093,
      "reward_std": 0.3174928646720946,
      "rewards/reward_func": 0.8736884651333093,
      "step": 3500
    },
    {
      "completion_length": 60.4325,
      "epoch": 0.72,
      "grad_norm": 12.125,
      "kl": 0.16217968232464045,
      "learning_rate": 4.039153688314146e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.9595573445409536,
      "reward_std": 0.34502996982075274,
      "rewards/reward_func": 0.9595573445409536,
      "step": 3600
    },
    {
      "completion_length": 63.43,
      "epoch": 0.74,
      "grad_norm": 12.9375,
      "kl": 0.290018264092505,
      "learning_rate": 3.969463130731183e-06,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.7895570612326265,
      "reward_std": 0.35805795643478633,
      "rewards/reward_func": 0.7895570612326265,
      "step": 3700
    },
    {
      "completion_length": 55.66,
      "epoch": 0.76,
      "grad_norm": 9.0,
      "kl": 0.19286280857399107,
      "learning_rate": 3.897982258676867e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8514765882119536,
      "reward_std": 0.40829260389087724,
      "rewards/reward_func": 0.8514765882119536,
      "step": 3800
    },
    {
      "completion_length": 64.0375,
      "epoch": 0.78,
      "grad_norm": 6.9375,
      "kl": 0.3412795978039503,
      "learning_rate": 3.824798160583012e-06,
      "loss": 0.0,
      "match_ratio": 0.9925,
      "reward": 0.9143854442238808,
      "reward_std": 0.3976023513358086,
      "rewards/reward_func": 0.9143854442238808,
      "step": 3900
    },
    {
      "completion_length": 63.335,
      "epoch": 0.8,
      "grad_norm": 11.375,
      "kl": 0.27103981951251627,
      "learning_rate": 3.7500000000000005e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.7887253789231181,
      "reward_std": 0.4162597674317658,
      "rewards/reward_func": 0.7887253789231181,
      "step": 4000
    },
    {
      "completion_length": 59.8575,
      "epoch": 0.82,
      "grad_norm": 6.59375,
      "kl": 0.7605254784226417,
      "learning_rate": 3.6736789069647273e-06,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.9478698487579823,
      "reward_std": 0.30987203000113367,
      "rewards/reward_func": 0.9478698487579823,
      "step": 4100
    },
    {
      "completion_length": 58.2175,
      "epoch": 0.84,
      "grad_norm": 12.3125,
      "kl": 1.3949576319474728,
      "learning_rate": 3.595927866972694e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.9541199389472603,
      "reward_std": 0.3034708809526637,
      "rewards/reward_func": 0.9541199389472603,
      "step": 4200
    },
    {
      "completion_length": 63.415,
      "epoch": 0.86,
      "grad_norm": 27.75,
      "kl": 0.45249753130599857,
      "learning_rate": 3.516841607689501e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8886630642414093,
      "reward_std": 0.36593056879937647,
      "rewards/reward_func": 0.8886630642414093,
      "step": 4300
    },
    {
      "completion_length": 55.02,
      "epoch": 0.88,
      "grad_norm": 11.25,
      "kl": 68.24156057231593,
      "learning_rate": 3.436516483539781e-06,
      "loss": 0.0068,
      "match_ratio": 1.0,
      "reward": 0.866313117146492,
      "reward_std": 0.35739596346393226,
      "rewards/reward_func": 0.866313117146492,
      "step": 4400
    },
    {
      "completion_length": 61.2375,
      "epoch": 0.9,
      "grad_norm": 21.5,
      "kl": 0.557325184418587,
      "learning_rate": 3.3550503583141726e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.845514679402113,
      "reward_std": 0.3533631762489676,
      "rewards/reward_func": 0.845514679402113,
      "step": 4500
    },
    {
      "completion_length": 61.4575,
      "epoch": 0.92,
      "grad_norm": 10.9375,
      "kl": 0.31026113393716515,
      "learning_rate": 3.272542485937369e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8414725087583065,
      "reward_std": 0.31504234885796906,
      "rewards/reward_func": 0.8414725087583065,
      "step": 4600
    },
    {
      "completion_length": 55.6725,
      "epoch": 0.94,
      "grad_norm": 15.875,
      "kl": 0.25192020772024987,
      "learning_rate": 3.189093389542498e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.934078385848552,
      "reward_std": 0.3096505870204419,
      "rewards/reward_func": 0.934078385848552,
      "step": 4700
    },
    {
      "completion_length": 57.29,
      "epoch": 0.96,
      "grad_norm": 8.75,
      "kl": 0.6216541412565857,
      "learning_rate": 3.1048047389991693e-06,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.8389100107550621,
      "reward_std": 0.3749863849021494,
      "rewards/reward_func": 0.8389100107550621,
      "step": 4800
    },
    {
      "completion_length": 61.2525,
      "epoch": 0.98,
      "grad_norm": 8.125,
      "kl": 0.342362194955349,
      "learning_rate": 3.019779227044398e-06,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.8242513693869113,
      "reward_std": 0.36022061900235713,
      "rewards/reward_func": 0.8242513693869113,
      "step": 4900
    },
    {
      "completion_length": 59.7125,
      "epoch": 1.0,
      "grad_norm": 17.5,
      "kl": 0.41126792770577597,
      "learning_rate": 2.9341204441673267e-06,
      "loss": 0.0,
      "match_ratio": 0.9925,
      "reward": 0.8274249080568552,
      "reward_std": 0.4075367634743452,
      "rewards/reward_func": 0.8274249080568552,
      "step": 5000
    },
    {
      "completion_length": 60.6025,
      "epoch": 1.02,
      "grad_norm": 12.375,
      "kl": 0.23068872857838868,
      "learning_rate": 2.847932752400164e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8065017646364868,
      "reward_std": 0.36464907992631196,
      "rewards/reward_func": 0.8065017646364868,
      "step": 5100
    },
    {
      "completion_length": 59.0775,
      "epoch": 1.04,
      "grad_norm": 20.25,
      "kl": 0.42417401013895867,
      "learning_rate": 2.761321158169134e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.7185550931096077,
      "reward_std": 0.40595400186255576,
      "rewards/reward_func": 0.7185550931096077,
      "step": 5200
    },
    {
      "completion_length": 63.42,
      "epoch": 1.06,
      "grad_norm": 23.875,
      "kl": 0.3724514145217836,
      "learning_rate": 2.6743911843603134e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.7712494351714849,
      "reward_std": 0.37587333597242834,
      "rewards/reward_func": 0.7712494351714849,
      "step": 5300
    },
    {
      "completion_length": 60.4,
      "epoch": 1.08,
      "grad_norm": 13.0,
      "kl": 0.41718232361599805,
      "learning_rate": 2.587248741756253e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8315356434136629,
      "reward_std": 0.35958164227195083,
      "rewards/reward_func": 0.8315356434136629,
      "step": 5400
    },
    {
      "completion_length": 64.535,
      "epoch": 1.1,
      "grad_norm": 13.0,
      "kl": 0.426719272416085,
      "learning_rate": 2.5e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8871817947924137,
      "reward_std": 0.35918335968628523,
      "rewards/reward_func": 0.8871817947924137,
      "step": 5500
    },
    {
      "completion_length": 63.825,
      "epoch": 1.12,
      "grad_norm": 12.6875,
      "kl": 10.818288787528873,
      "learning_rate": 2.4127512582437486e-06,
      "loss": 0.0011,
      "match_ratio": 0.9925,
      "reward": 0.8072922784090042,
      "reward_std": 0.41990237571299077,
      "rewards/reward_func": 0.8072922784090042,
      "step": 5600
    },
    {
      "completion_length": 61.5425,
      "epoch": 1.1400000000000001,
      "grad_norm": 8.5625,
      "kl": 1.7851288786903023,
      "learning_rate": 2.325608815639687e-06,
      "loss": 0.0002,
      "match_ratio": 0.9975,
      "reward": 0.7871765466406941,
      "reward_std": 0.3983499974012375,
      "rewards/reward_func": 0.7871765466406941,
      "step": 5700
    },
    {
      "completion_length": 64.99,
      "epoch": 1.16,
      "grad_norm": 83.5,
      "kl": 1.0691816475684754,
      "learning_rate": 2.238678841830867e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.9227658536192029,
      "reward_std": 0.3650888724066317,
      "rewards/reward_func": 0.9227658536192029,
      "step": 5800
    },
    {
      "completion_length": 65.0375,
      "epoch": 1.18,
      "grad_norm": 9.8125,
      "kl": 0.38561805644072594,
      "learning_rate": 2.1520672475998374e-06,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8525355974957347,
      "reward_std": 0.36985819303430617,
      "rewards/reward_func": 0.8525355974957347,
      "step": 5900
    },
    {
      "completion_length": 62.8275,
      "epoch": 1.2,
      "grad_norm": 9.3125,
      "kl": 0.3610994891449809,
      "learning_rate": 2.0658795558326745e-06,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 1.0021917837299406,
      "reward_std": 0.3685089880321175,
      "rewards/reward_func": 1.0021917837299406,
      "step": 6000
    },
    {
      "completion_length": 67.415,
      "epoch": 1.22,
      "grad_norm": 14.0,
      "kl": 0.48120407085865735,
      "learning_rate": 1.9802207729556023e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.980966807603836,
      "reward_std": 0.33038541514426467,
      "rewards/reward_func": 0.980966807603836,
      "step": 6100
    },
    {
      "completion_length": 61.275,
      "epoch": 1.24,
      "grad_norm": 17.125,
      "kl": 1.394161350093782,
      "learning_rate": 1.895195261000831e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8998113541305065,
      "reward_std": 0.37138622866943477,
      "rewards/reward_func": 0.8998113541305065,
      "step": 6200
    },
    {
      "completion_length": 61.4575,
      "epoch": 1.26,
      "grad_norm": 9.5625,
      "kl": 0.4891889825835824,
      "learning_rate": 1.8109066104575023e-06,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.7821180200204253,
      "reward_std": 0.4117224833089858,
      "rewards/reward_func": 0.7821180200204253,
      "step": 6300
    },
    {
      "completion_length": 62.535,
      "epoch": 1.28,
      "grad_norm": 17.75,
      "kl": 0.701120622754097,
      "learning_rate": 1.7274575140626318e-06,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.8505088457465172,
      "reward_std": 0.35196221828460694,
      "rewards/reward_func": 0.8505088457465172,
      "step": 6400
    },
    {
      "completion_length": 61.0425,
      "epoch": 1.3,
      "grad_norm": 10.4375,
      "kl": 0.8444961504405364,
      "learning_rate": 1.6449496416858285e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8533294384181499,
      "reward_std": 0.35340342290699484,
      "rewards/reward_func": 0.8533294384181499,
      "step": 6500
    },
    {
      "completion_length": 66.6525,
      "epoch": 1.32,
      "grad_norm": 16.375,
      "kl": 0.6673866561800241,
      "learning_rate": 1.56348351646022e-06,
      "loss": 0.0001,
      "match_ratio": 0.9925,
      "reward": 0.962776445467025,
      "reward_std": 0.39187521073035897,
      "rewards/reward_func": 0.962776445467025,
      "step": 6600
    },
    {
      "completion_length": 64.05,
      "epoch": 1.34,
      "grad_norm": 15.25,
      "kl": 0.5449268382415176,
      "learning_rate": 1.4831583923105e-06,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.9354583528265357,
      "reward_std": 0.3569179131626152,
      "rewards/reward_func": 0.9354583528265357,
      "step": 6700
    },
    {
      "completion_length": 64.9625,
      "epoch": 1.3599999999999999,
      "grad_norm": 12.875,
      "kl": 0.9086799253150821,
      "learning_rate": 1.4040721330273063e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8598837627470494,
      "reward_std": 0.36035713417921217,
      "rewards/reward_func": 0.8598837627470494,
      "step": 6800
    },
    {
      "completion_length": 65.365,
      "epoch": 1.38,
      "grad_norm": 16.5,
      "kl": 0.5496124785766006,
      "learning_rate": 1.3263210930352737e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8178367885202169,
      "reward_std": 0.37116443024016915,
      "rewards/reward_func": 0.8178367885202169,
      "step": 6900
    },
    {
      "completion_length": 62.0475,
      "epoch": 1.4,
      "grad_norm": 26.875,
      "kl": 0.549699901342392,
      "learning_rate": 1.2500000000000007e-06,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.9533282884210348,
      "reward_std": 0.3430164767615497,
      "rewards/reward_func": 0.9533282884210348,
      "step": 7000
    },
    {
      "completion_length": 60.79,
      "epoch": 1.42,
      "grad_norm": 13.25,
      "kl": 0.5086184279620647,
      "learning_rate": 1.1752018394169882e-06,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.8821756513416767,
      "reward_std": 0.3568952218815684,
      "rewards/reward_func": 0.8821756513416767,
      "step": 7100
    },
    {
      "completion_length": 62.3475,
      "epoch": 1.44,
      "grad_norm": 21.125,
      "kl": 0.8897788706421852,
      "learning_rate": 1.1020177413231334e-06,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.934985687956214,
      "reward_std": 0.32632746720686556,
      "rewards/reward_func": 0.934985687956214,
      "step": 7200
    },
    {
      "completion_length": 56.98,
      "epoch": 1.46,
      "grad_norm": 35.25,
      "kl": 0.7370296374708414,
      "learning_rate": 1.0305368692688175e-06,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8605528651922941,
      "reward_std": 0.3732724652206525,
      "rewards/reward_func": 0.8605528651922941,
      "step": 7300
    },
    {
      "completion_length": 57.71,
      "epoch": 1.48,
      "grad_norm": 14.5,
      "kl": 0.7560949631407857,
      "learning_rate": 9.608463116858544e-07,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.8051678024046123,
      "reward_std": 0.3447662947047502,
      "rewards/reward_func": 0.8051678024046123,
      "step": 7400
    },
    {
      "completion_length": 69.1925,
      "epoch": 1.5,
      "grad_norm": 29.5,
      "kl": 0.6943446175381541,
      "learning_rate": 8.930309757836517e-07,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8813337843865157,
      "reward_std": 0.3512991077173501,
      "rewards/reward_func": 0.8813337843865157,
      "step": 7500
    },
    {
      "completion_length": 62.1225,
      "epoch": 1.52,
      "grad_norm": 11.25,
      "kl": 0.3682367965579033,
      "learning_rate": 8.271734841028553e-07,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.8769916776567698,
      "reward_std": 0.35788299994543193,
      "rewards/reward_func": 0.8769916776567698,
      "step": 7600
    },
    {
      "completion_length": 62.5475,
      "epoch": 1.54,
      "grad_norm": 10.125,
      "kl": 0.35848211450036616,
      "learning_rate": 7.633540738525066e-07,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.9400549785792828,
      "reward_std": 0.36338255695067345,
      "rewards/reward_func": 0.9400549785792828,
      "step": 7700
    },
    {
      "completion_length": 61.32,
      "epoch": 1.56,
      "grad_norm": 13.625,
      "kl": 0.5323085347935558,
      "learning_rate": 7.016504991533727e-07,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.8762814123183489,
      "reward_std": 0.35265143546042965,
      "rewards/reward_func": 0.8762814123183489,
      "step": 7800
    },
    {
      "completion_length": 66.315,
      "epoch": 1.58,
      "grad_norm": 18.375,
      "kl": 0.40489839322865007,
      "learning_rate": 6.421379363065142e-07,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.8989605332165956,
      "reward_std": 0.3447486224025488,
      "rewards/reward_func": 0.8989605332165956,
      "step": 7900
    },
    {
      "completion_length": 60.5325,
      "epoch": 1.6,
      "grad_norm": 11.75,
      "kl": 0.690227730597835,
      "learning_rate": 5.848888922025553e-07,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.8210568431764841,
      "reward_std": 0.3149324245750904,
      "rewards/reward_func": 0.8210568431764841,
      "step": 8000
    },
    {
      "completion_length": 66.7825,
      "epoch": 1.62,
      "grad_norm": 13.4375,
      "kl": 0.516472494918853,
      "learning_rate": 5.299731159831953e-07,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.8683985948190093,
      "reward_std": 0.345150127671659,
      "rewards/reward_func": 0.8683985948190093,
      "step": 8100
    },
    {
      "completion_length": 65.72,
      "epoch": 1.6400000000000001,
      "grad_norm": 12.875,
      "kl": 0.4457222482562065,
      "learning_rate": 4.774575140626317e-07,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.7730556976422668,
      "reward_std": 0.41231788201257585,
      "rewards/reward_func": 0.7730556976422668,
      "step": 8200
    },
    {
      "completion_length": 64.4875,
      "epoch": 1.6600000000000001,
      "grad_norm": 9.6875,
      "kl": 0.5901041788049042,
      "learning_rate": 4.27406068612396e-07,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.8864369177818299,
      "reward_std": 0.3609216751717031,
      "rewards/reward_func": 0.8864369177818299,
      "step": 8300
    },
    {
      "completion_length": 59.98,
      "epoch": 1.6800000000000002,
      "grad_norm": 30.5,
      "kl": 0.3067289407923818,
      "learning_rate": 3.798797596089351e-07,
      "loss": 0.0,
      "match_ratio": 0.9975,
      "reward": 0.8906753876060247,
      "reward_std": 0.2947716296184808,
      "rewards/reward_func": 0.8906753876060247,
      "step": 8400
    },
    {
      "completion_length": 68.6625,
      "epoch": 1.7,
      "grad_norm": 24.0,
      "kl": 2.096909821406007,
      "learning_rate": 3.3493649053890325e-07,
      "loss": 0.0002,
      "match_ratio": 1.0,
      "reward": 0.9327631609933451,
      "reward_std": 0.3544263231381774,
      "rewards/reward_func": 0.9327631609933451,
      "step": 8500
    },
    {
      "completion_length": 66.0125,
      "epoch": 1.72,
      "grad_norm": 10.9375,
      "kl": 0.5123670964688063,
      "learning_rate": 2.9263101785268253e-07,
      "loss": 0.0001,
      "match_ratio": 0.995,
      "reward": 0.9494618388265371,
      "reward_std": 0.35082788893952965,
      "rewards/reward_func": 0.9494618388265371,
      "step": 8600
    },
    {
      "completion_length": 57.8475,
      "epoch": 1.74,
      "grad_norm": 15.8125,
      "kl": 1.2104839562997223,
      "learning_rate": 2.53014884252083e-07,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.9292678725533188,
      "reward_std": 0.3112880502641201,
      "rewards/reward_func": 0.9292678725533188,
      "step": 8700
    },
    {
      "completion_length": 63.66,
      "epoch": 1.76,
      "grad_norm": 10.9375,
      "kl": 0.5559730716235936,
      "learning_rate": 2.1613635589349756e-07,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.9810863409936428,
      "reward_std": 0.36776383105432614,
      "rewards/reward_func": 0.9810863409936428,
      "step": 8800
    },
    {
      "completion_length": 59.8675,
      "epoch": 1.78,
      "grad_norm": 13.25,
      "kl": 29.805520134083928,
      "learning_rate": 1.8204036358303173e-07,
      "loss": 0.003,
      "match_ratio": 1.0,
      "reward": 0.8572433185577393,
      "reward_std": 0.3437820218596607,
      "rewards/reward_func": 0.8572433185577393,
      "step": 8900
    },
    {
      "completion_length": 60.3075,
      "epoch": 1.8,
      "grad_norm": 9.1875,
      "kl": 0.8873812770657241,
      "learning_rate": 1.507684480352292e-07,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.919341038018465,
      "reward_std": 0.33141862623393537,
      "rewards/reward_func": 0.919341038018465,
      "step": 9000
    },
    {
      "completion_length": 62.265,
      "epoch": 1.8199999999999998,
      "grad_norm": 12.125,
      "kl": 0.6630043520405888,
      "learning_rate": 1.223587092621162e-07,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 1.029205017723143,
      "reward_std": 0.3310457341000438,
      "rewards/reward_func": 1.029205017723143,
      "step": 9100
    },
    {
      "completion_length": 62.0575,
      "epoch": 1.8399999999999999,
      "grad_norm": 11.0,
      "kl": 0.5343834590911866,
      "learning_rate": 9.684576015420277e-08,
      "loss": 0.0001,
      "match_ratio": 0.9975,
      "reward": 0.8843235304579139,
      "reward_std": 0.35643110671080647,
      "rewards/reward_func": 0.8843235304579139,
      "step": 9200
    },
    {
      "completion_length": 64.51,
      "epoch": 1.8599999999999999,
      "grad_norm": 13.0,
      "kl": 0.47331944581121205,
      "learning_rate": 7.426068431000883e-08,
      "loss": 0.0,
      "match_ratio": 1.0,
      "reward": 0.8205329022929072,
      "reward_std": 0.33430186320096256,
      "rewards/reward_func": 0.8205329022929072,
      "step": 9300
    },
    {
      "completion_length": 64.395,
      "epoch": 1.88,
      "grad_norm": 42.0,
      "kl": 1.8421870478987694,
      "learning_rate": 5.463099816548578e-08,
      "loss": 0.0002,
      "match_ratio": 0.9975,
      "reward": 0.9012074111029506,
      "reward_std": 0.33403355406597257,
      "rewards/reward_func": 0.9012074111029506,
      "step": 9400
    },
    {
      "completion_length": 61.4425,
      "epoch": 1.9,
      "grad_norm": 12.0,
      "kl": 0.5331659988686442,
      "learning_rate": 3.798061746947995e-08,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.7935629660636186,
      "reward_std": 0.3497585416212678,
      "rewards/reward_func": 0.7935629660636186,
      "step": 9500
    },
    {
      "completion_length": 51.77,
      "epoch": 1.92,
      "grad_norm": 10.875,
      "kl": 0.7931823456101119,
      "learning_rate": 2.4329828146074096e-08,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.9315874481201172,
      "reward_std": 0.44919231578707697,
      "rewards/reward_func": 0.9315874481201172,
      "step": 9600
    },
    {
      "completion_length": 57.5275,
      "epoch": 1.94,
      "grad_norm": 13.9375,
      "kl": 0.6070253856666387,
      "learning_rate": 1.3695261579316776e-08,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.9431364990770816,
      "reward_std": 0.3222667661588639,
      "rewards/reward_func": 0.9431364990770816,
      "step": 9700
    },
    {
      "completion_length": 59.37,
      "epoch": 1.96,
      "grad_norm": 14.875,
      "kl": 0.5334895004890859,
      "learning_rate": 6.089874350439507e-09,
      "loss": 0.0001,
      "match_ratio": 1.0,
      "reward": 0.9747491884231567,
      "reward_std": 0.29319573145825417,
      "rewards/reward_func": 0.9747491884231567,
      "step": 9800
    },
    {
      "completion_length": 63.385,
      "epoch": 1.98,
      "grad_norm": 12.125,
      "kl": 0.4961644561961293,
      "learning_rate": 1.5229324522605949e-09,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.8896720813587308,
      "reward_std": 0.4128014264255762,
      "rewards/reward_func": 0.8896720813587308,
      "step": 9900
    },
    {
      "completion_length": 67.28,
      "epoch": 2.0,
      "grad_norm": 10.8125,
      "kl": 0.30117323972284793,
      "learning_rate": 0.0,
      "loss": 0.0,
      "match_ratio": 0.995,
      "reward": 0.7791323178261519,
      "reward_std": 0.35405952845700084,
      "rewards/reward_func": 0.7791323178261519,
      "step": 10000
    }
  ],
  "logging_steps": 100,
  "max_steps": 10000,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 2,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}