{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 7.5,
  "eval_steps": 10,
  "global_step": 30,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 25.59375,
      "default/reward": 0.5906250178813934,
      "default/reward_std": 0.5985057950019836,
      "default/rewards/llm_judge_reward_function": 0.6000000089406967,
      "epoch": 0.25,
      "grad_norm": 4.563571929931641,
      "kl": 0.0,
      "learning_rate": 5e-06,
      "loss": 0.0,
      "step": 1
    },
    {
      "completion_length": 29.46875,
      "default/reward": 0.6906249821186066,
      "default/reward_std": 0.8958407640457153,
      "default/rewards/llm_judge_reward_function": 0.8499999940395355,
      "epoch": 0.5,
      "grad_norm": 2.090252637863159,
      "kl": 0.00035738608858082443,
      "learning_rate": 4.999143312438893e-06,
      "loss": 0.0068,
      "step": 2
    },
    {
      "completion_length": 32.25,
      "default/reward": 0.7562499940395355,
      "default/reward_std": 0.6679378151893616,
      "default/rewards/llm_judge_reward_function": 0.42500001192092896,
      "epoch": 0.75,
      "grad_norm": 3.4133903980255127,
      "kl": 0.0035452890442684293,
      "learning_rate": 4.9965738368864345e-06,
      "loss": 0.0,
      "step": 3
    },
    {
      "completion_length": 23.416667938232422,
      "default/reward": 0.734375,
      "default/reward_std": 0.7354854047298431,
      "default/rewards/llm_judge_reward_function": 0.5,
      "epoch": 1.0,
      "grad_norm": 4.797443389892578,
      "kl": 0.003295191505458206,
      "learning_rate": 4.992293334332821e-06,
      "loss": 0.0,
      "step": 4
    },
    {
      "completion_length": 27.9375,
      "default/reward": 0.546875,
      "default/reward_std": 0.6988737881183624,
      "default/rewards/llm_judge_reward_function": 0.925000011920929,
      "epoch": 1.25,
      "grad_norm": 2.957021713256836,
      "kl": 0.013654953800141811,
      "learning_rate": 4.986304738420684e-06,
      "loss": 0.0001,
      "step": 5
    },
    {
      "completion_length": 33.71875,
      "default/reward": 0.653124988079071,
      "default/reward_std": 0.69887375831604,
      "default/rewards/llm_judge_reward_function": 0.925000011920929,
      "epoch": 1.5,
      "grad_norm": 2.791567802429199,
      "kl": 0.015739883296191692,
      "learning_rate": 4.978612153434527e-06,
      "loss": 0.0002,
      "step": 6
    },
    {
      "completion_length": 30.53125,
      "default/reward": 0.768750011920929,
      "default/reward_std": 0.8825825154781342,
      "default/rewards/llm_judge_reward_function": 0.6749999970197678,
      "epoch": 1.75,
      "grad_norm": 2.669797658920288,
      "kl": 0.021481021773070097,
      "learning_rate": 4.9692208514878445e-06,
      "loss": 0.0002,
      "step": 7
    },
    {
      "completion_length": 29.791666984558105,
      "default/reward": 0.6937500238418579,
      "default/reward_std": 0.7657932043075562,
      "default/rewards/llm_judge_reward_function": 0.675000011920929,
      "epoch": 2.0,
      "grad_norm": 2.478447437286377,
      "kl": 0.04025824833661318,
      "learning_rate": 4.958137268909887e-06,
      "loss": 0.0004,
      "step": 8
    },
    {
      "completion_length": 39.15625,
      "default/reward": 0.737500011920929,
      "default/reward_std": 0.6673097014427185,
      "default/rewards/llm_judge_reward_function": 0.675000011920929,
      "epoch": 2.25,
      "grad_norm": 2.5778961181640625,
      "kl": 0.026319073513150215,
      "learning_rate": 4.9453690018345144e-06,
      "loss": 0.0003,
      "step": 9
    },
    {
      "completion_length": 28.46875,
      "default/reward": 0.7218749821186066,
      "default/reward_std": 0.7424398064613342,
      "default/rewards/llm_judge_reward_function": 0.5,
      "epoch": 2.5,
      "grad_norm": 4.112423896789551,
      "kl": 0.09163418784737587,
      "learning_rate": 4.930924800994192e-06,
      "loss": 0.0009,
      "step": 10
    },
    {
      "epoch": 2.5,
      "eval_default/reward": 0.8224999904632568,
      "eval_default/reward_std": 0.6798096895217896,
      "eval_default/rewards/llm_judge_reward_function": 0.7399999976158143,
      "eval_loss": 0.0,
      "eval_runtime": 432.2348,
      "eval_samples_per_second": 0.046,
      "eval_steps_per_second": 0.007,
      "step": 10
    },
    {
      "completion_length": 25.09375,
      "default/reward": 0.893750011920929,
      "default/reward_std": 0.5877601504325867,
      "default/rewards/llm_judge_reward_function": 0.8500000238418579,
      "epoch": 2.75,
      "grad_norm": 2.8137168884277344,
      "kl": 0.05960903689265251,
      "learning_rate": 4.914814565722671e-06,
      "loss": 0.0006,
      "step": 11
    },
    {
      "completion_length": 33.20833396911621,
      "default/reward": 0.8531250059604645,
      "default/reward_std": 0.8522747457027435,
      "default/rewards/llm_judge_reward_function": 0.8499999940395355,
      "epoch": 3.0,
      "grad_norm": 1.4571622610092163,
      "kl": 0.06877592019736767,
      "learning_rate": 4.897049337170483e-06,
      "loss": 0.0007,
      "step": 12
    },
    {
      "completion_length": 30.59375,
      "default/reward": 0.7874999940395355,
      "default/reward_std": 0.6849873661994934,
      "default/rewards/llm_judge_reward_function": 0.925000011920929,
      "epoch": 3.25,
      "grad_norm": 2.8570544719696045,
      "kl": 0.05570350959897041,
      "learning_rate": 4.8776412907378845e-06,
      "loss": 0.0006,
      "step": 13
    },
    {
      "completion_length": 39.25,
      "default/reward": 0.7281250357627869,
      "default/reward_std": 0.7159232497215271,
      "default/rewards/llm_judge_reward_function": 0.925000011920929,
      "epoch": 3.5,
      "grad_norm": 2.8516972064971924,
      "kl": 0.10984514653682709,
      "learning_rate": 4.856603727730446e-06,
      "loss": 0.0011,
      "step": 14
    },
    {
      "completion_length": 29.09375,
      "default/reward": 0.824999988079071,
      "default/reward_std": 0.9084708988666534,
      "default/rewards/llm_judge_reward_function": 0.75,
      "epoch": 3.75,
      "grad_norm": 3.0547258853912354,
      "kl": 0.177144393324852,
      "learning_rate": 4.833951066243004e-06,
      "loss": 0.0018,
      "step": 15
    },
    {
      "completion_length": 29.500000953674316,
      "default/reward": 0.8812499940395355,
      "default/reward_std": 0.8649048507213593,
      "default/rewards/llm_judge_reward_function": 1.0,
      "epoch": 4.0,
      "grad_norm": 1.7537461519241333,
      "kl": 0.08436176925897598,
      "learning_rate": 4.809698831278217e-06,
      "loss": 0.0008,
      "step": 16
    },
    {
      "completion_length": 35.625,
      "default/reward": 0.8687499761581421,
      "default/reward_std": 0.8200824856758118,
      "default/rewards/llm_judge_reward_function": 1.0,
      "epoch": 4.25,
      "grad_norm": 3.1518936157226562,
      "kl": 0.09947774186730385,
      "learning_rate": 4.783863644106502e-06,
      "loss": 0.001,
      "step": 17
    },
    {
      "completion_length": 43.34375,
      "default/reward": 0.5843750238418579,
      "default/reward_std": 0.715295135974884,
      "default/rewards/llm_judge_reward_function": 0.5999999940395355,
      "epoch": 4.5,
      "grad_norm": 3.3874547481536865,
      "kl": 0.11295061185956001,
      "learning_rate": 4.7564632108746524e-06,
      "loss": 0.0011,
      "step": 18
    },
    {
      "completion_length": 32.34375,
      "default/reward": 0.734375,
      "default/reward_std": 0.7973572611808777,
      "default/rewards/llm_judge_reward_function": 1.0,
      "epoch": 4.75,
      "grad_norm": 2.7637128829956055,
      "kl": 0.1756831854581833,
      "learning_rate": 4.72751631047092e-06,
      "loss": 0.0018,
      "step": 19
    },
    {
      "completion_length": 39.45833396911621,
      "default/reward": 0.7562500238418579,
      "default/reward_std": 0.8383883237838745,
      "default/rewards/llm_judge_reward_function": 0.5249999910593033,
      "epoch": 5.0,
      "grad_norm": 1.7747427225112915,
      "kl": 0.13021686673164368,
      "learning_rate": 4.697042781654913e-06,
      "loss": 0.0013,
      "step": 20
    },
    {
      "epoch": 5.0,
      "eval_default/reward": 0.8600000143051147,
      "eval_default/reward_std": 0.7343502879142761,
      "eval_default/rewards/llm_judge_reward_function": 0.8799999952316284,
      "eval_loss": 0.0,
      "eval_runtime": 431.4917,
      "eval_samples_per_second": 0.046,
      "eval_steps_per_second": 0.007,
      "step": 20
    },
    {
      "completion_length": 28.53125,
      "default/reward": 0.6343749910593033,
      "default/reward_std": 0.6717291176319122,
      "default/rewards/llm_judge_reward_function": 0.8500000238418579,
      "epoch": 5.25,
      "grad_norm": 3.542097330093384,
      "kl": 0.15948720276355743,
      "learning_rate": 4.665063509461098e-06,
      "loss": 0.0016,
      "step": 21
    },
    {
      "completion_length": 31.21875,
      "default/reward": 0.8375000059604645,
      "default/reward_std": 0.8649048805236816,
      "default/rewards/llm_judge_reward_function": 1.0,
      "epoch": 5.5,
      "grad_norm": 2.465200662612915,
      "kl": 0.16199305281043053,
      "learning_rate": 4.631600410885231e-06,
      "loss": 0.0016,
      "step": 22
    },
    {
      "completion_length": 25.5625,
      "default/reward": 0.934374988079071,
      "default/reward_std": 0.9507582485675812,
      "default/rewards/llm_judge_reward_function": 0.8499999940395355,
      "epoch": 5.75,
      "grad_norm": 0.9499327540397644,
      "kl": 0.25974033027887344,
      "learning_rate": 4.596676419863561e-06,
      "loss": -0.015,
      "step": 23
    },
    {
      "completion_length": 24.625,
      "default/reward": 0.7781250178813934,
      "default/reward_std": 0.6982456147670746,
      "default/rewards/llm_judge_reward_function": 0.5,
      "epoch": 6.0,
      "grad_norm": 3.659409999847412,
      "kl": 0.317486435174942,
      "learning_rate": 4.560315471555039e-06,
      "loss": 0.0032,
      "step": 24
    },
    {
      "completion_length": 24.1875,
      "default/reward": 0.7343750298023224,
      "default/reward_std": 0.5915514230728149,
      "default/rewards/llm_judge_reward_function": 0.925000011920929,
      "epoch": 6.25,
      "grad_norm": 9.055826187133789,
      "kl": 0.19061780348420143,
      "learning_rate": 4.522542485937369e-06,
      "loss": 0.0019,
      "step": 25
    },
    {
      "completion_length": 34.84375,
      "default/reward": 0.875,
      "default/reward_std": 0.784727156162262,
      "default/rewards/llm_judge_reward_function": 0.8499999940395355,
      "epoch": 6.5,
      "grad_norm": 3.1907670497894287,
      "kl": 0.30184144526720047,
      "learning_rate": 4.4833833507280884e-06,
      "loss": 0.003,
      "step": 26
    },
    {
      "completion_length": 22.625,
      "default/reward": 0.828125,
      "default/reward_std": 0.8522747457027435,
      "default/rewards/llm_judge_reward_function": 0.42500001192092896,
      "epoch": 6.75,
      "grad_norm": 2.9145781993865967,
      "kl": 0.39268461614847183,
      "learning_rate": 4.442864903642428e-06,
      "loss": -0.0043,
      "step": 27
    },
    {
      "completion_length": 37.29166793823242,
      "default/reward": 0.7187500298023224,
      "default/reward_std": 0.6666815280914307,
      "default/rewards/llm_judge_reward_function": 1.0,
      "epoch": 7.0,
      "grad_norm": 4.502452850341797,
      "kl": 0.2312646582722664,
      "learning_rate": 4.401014914000078e-06,
      "loss": 0.0023,
      "step": 28
    },
    {
      "completion_length": 27.46875,
      "default/reward": 0.796875,
      "default/reward_std": 0.6988737881183624,
      "default/rewards/llm_judge_reward_function": 0.925000011920929,
      "epoch": 7.25,
      "grad_norm": 3.6535165309906006,
      "kl": 0.2524450868368149,
      "learning_rate": 4.357862063693486e-06,
      "loss": 0.0025,
      "step": 29
    },
    {
      "completion_length": 26.90625,
      "default/reward": 0.84375,
      "default/reward_std": 0.8030329942703247,
      "default/rewards/llm_judge_reward_function": 0.42500001192092896,
      "epoch": 7.5,
      "grad_norm": 1.068318247795105,
      "kl": 0.20408491045236588,
      "learning_rate": 4.313435927530719e-06,
      "loss": 0.002,
      "step": 30
    },
    {
      "epoch": 7.5,
      "eval_default/reward": 0.9025000095367431,
      "eval_default/reward_std": 0.7242461919784546,
      "eval_default/rewards/llm_judge_reward_function": 0.9399999976158142,
      "eval_loss": 0.0,
      "eval_runtime": 422.8471,
      "eval_samples_per_second": 0.047,
      "eval_steps_per_second": 0.007,
      "step": 30
    }
  ],
  "logging_steps": 1,
  "max_steps": 120,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 30,
  "save_steps": 10,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}