{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.0002,
  "eval_steps": 500,
  "global_step": 10,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.90625,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 16.0,
      "completions/mean_length": 15.59375,
      "completions/mean_terminated_length": 11.666666984558105,
      "completions/min_length": 9.0,
      "completions/min_terminated_length": 9.0,
      "entropy": 0.45986470580101013,
      "epoch": 2e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5555855631828308,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.0036,
      "num_tokens": 53302.0,
      "reward": -20.565004348754883,
      "reward_std": 12.108231544494629,
      "rewards/rollout_reward_func/mean": -20.565004348754883,
      "rewards/rollout_reward_func/std": 12.108230590820312,
      "sampling/importance_sampling_ratio/max": 1.2069631814956665,
      "sampling/importance_sampling_ratio/mean": 1.0164316892623901,
      "sampling/importance_sampling_ratio/min": 0.691445529460907,
      "sampling/sampling_logp_difference/max": 0.14298677444458008,
      "sampling/sampling_logp_difference/mean": 0.01139700785279274,
      "step": 1,
      "step_time": 13.593145795000055
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 0.45986470580101013,
      "epoch": 4e-05,
      "grad_norm": 0.5478876829147339,
      "kl": 0.0,
      "learning_rate": 8.571428571428574e-08,
      "loss": -0.0036,
      "step": 2,
      "step_time": 3.5665654679996805
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0031250000465661287,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0031250000465661287,
      "completions/clipped_ratio": 0.75,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 16.0,
      "completions/mean_length": 14.4375,
      "completions/mean_terminated_length": 9.75,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4573390781879425,
      "epoch": 6e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4110209047794342,
      "kl": 0.00046882770402589813,
      "learning_rate": 1.7142857142857149e-07,
      "loss": 0.1188,
      "num_tokens": 106349.0,
      "reward": -22.559879302978516,
      "reward_std": 14.563770294189453,
      "rewards/rollout_reward_func/mean": -22.559879302978516,
      "rewards/rollout_reward_func/std": 14.563770294189453,
      "sampling/importance_sampling_ratio/max": 1.2753626108169556,
      "sampling/importance_sampling_ratio/mean": 0.9817989468574524,
      "sampling/importance_sampling_ratio/min": 0.8001569509506226,
      "sampling/sampling_logp_difference/max": 0.2265012264251709,
      "sampling/sampling_logp_difference/mean": 0.012659025378525257,
      "step": 3,
      "step_time": 13.572802005000085
    },
    {
      "clip_ratio/high_max": 0.0078125,
      "clip_ratio/high_mean": 0.001953125,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "entropy": 0.45671091973781586,
      "epoch": 8e-05,
      "grad_norm": 0.3884861469268799,
      "kl": 0.00040267666918225586,
      "learning_rate": 2.571428571428572e-07,
      "loss": 0.1171,
      "step": 4,
      "step_time": 3.672753089998878
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.78125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 13.0,
      "completions/mean_length": 14.09375,
      "completions/mean_terminated_length": 7.285714626312256,
      "completions/min_length": 4.0,
      "completions/min_terminated_length": 4.0,
      "entropy": 0.4637308567762375,
      "epoch": 0.0001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5923936367034912,
      "kl": 0.00045897291784058325,
      "learning_rate": 3.4285714285714297e-07,
      "loss": 0.1461,
      "num_tokens": 158201.0,
      "reward": -23.239702224731445,
      "reward_std": 11.33220386505127,
      "rewards/rollout_reward_func/mean": -23.239702224731445,
      "rewards/rollout_reward_func/std": 11.332201957702637,
      "sampling/importance_sampling_ratio/max": 1.179511308670044,
      "sampling/importance_sampling_ratio/mean": 0.9810092449188232,
      "sampling/importance_sampling_ratio/min": 0.7553079128265381,
      "sampling/sampling_logp_difference/max": 0.19478893280029297,
      "sampling/sampling_logp_difference/mean": 0.012187635526061058,
      "step": 5,
      "step_time": 12.394922012998904
    },
    {
      "epoch": 0.0001,
      "eval_clip_ratio/high_max": 0.0,
      "eval_clip_ratio/high_mean": 0.0,
      "eval_clip_ratio/low_mean": 0.0,
      "eval_clip_ratio/low_min": 0.0,
      "eval_clip_ratio/region_mean": 0.0,
      "eval_completions/clipped_ratio": 0.85,
      "eval_completions/max_length": 16.0,
      "eval_completions/max_terminated_length": 5.4,
      "eval_completions/mean_length": 15.225,
      "eval_completions/mean_terminated_length": 5.15,
      "eval_completions/min_length": 12.9,
      "eval_completions/min_terminated_length": 4.9,
      "eval_entropy": 0.45741056799888613,
      "eval_frac_reward_zero_std": 0.0,
      "eval_kl": 0.00042195062269456685,
      "eval_loss": 0.004154330585151911,
      "eval_num_tokens": 158201.0,
      "eval_reward": -25.101631736755373,
      "eval_reward_std": 13.229044437408447,
      "eval_rewards/rollout_reward_func/mean": -25.101631736755373,
      "eval_rewards/rollout_reward_func/std": 13.2290442943573,
      "eval_runtime": 13.4386,
      "eval_samples_per_second": 0.744,
      "eval_sampling/importance_sampling_ratio/max": 1.112263822555542,
      "eval_sampling/importance_sampling_ratio/mean": 0.9942494869232178,
      "eval_sampling/importance_sampling_ratio/min": 0.8756316006183624,
      "eval_sampling/sampling_logp_difference/max": 0.1471207022666931,
      "eval_sampling/sampling_logp_difference/mean": 0.013286880496889353,
      "eval_steps_per_second": 0.223,
      "step": 5
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 0.46300478279590607,
      "epoch": 0.00012,
      "grad_norm": 0.5754485726356506,
      "kl": 0.0004794619817403145,
      "learning_rate": 4.2857142857142867e-07,
      "loss": 0.1461,
      "step": 6,
      "step_time": 3.5247847910004566
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.78125,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 14.0,
      "completions/mean_length": 14.6875,
      "completions/mean_terminated_length": 10.0,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4924817681312561,
      "epoch": 0.00014,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5342097282409668,
      "kl": 0.00047566716602887027,
      "learning_rate": 5.142857142857144e-07,
      "loss": 0.0788,
      "num_tokens": 211022.0,
      "reward": -21.68277359008789,
      "reward_std": 14.98656940460205,
      "rewards/rollout_reward_func/mean": -21.68277359008789,
      "rewards/rollout_reward_func/std": 14.986568450927734,
      "sampling/importance_sampling_ratio/max": 1.2211930751800537,
      "sampling/importance_sampling_ratio/mean": 0.9879010319709778,
      "sampling/importance_sampling_ratio/min": 0.7968342304229736,
      "sampling/sampling_logp_difference/max": 0.2077500820159912,
      "sampling/sampling_logp_difference/mean": 0.015078969299793243,
      "step": 7,
      "step_time": 13.125986247000583
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.001953125,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.001953125,
      "entropy": 0.4939192831516266,
      "epoch": 0.00016,
      "grad_norm": 0.5269240140914917,
      "kl": 0.0005614961846731603,
      "learning_rate": 6.000000000000002e-07,
      "loss": 0.078,
      "step": 8,
      "step_time": 4.57110312200075
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.84375,
      "completions/max_length": 16.0,
      "completions/max_terminated_length": 16.0,
      "completions/mean_length": 15.53125,
      "completions/mean_terminated_length": 13.0,
      "completions/min_length": 3.0,
      "completions/min_terminated_length": 3.0,
      "entropy": 0.4650999382138252,
      "epoch": 0.00018,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4413743019104004,
      "kl": 0.0005437940853880718,
      "learning_rate": 6.857142857142859e-07,
      "loss": 0.0419,
      "num_tokens": 262651.0,
      "reward": -24.077335357666016,
      "reward_std": 11.67958927154541,
      "rewards/rollout_reward_func/mean": -24.077335357666016,
      "rewards/rollout_reward_func/std": 11.67958927154541,
      "sampling/importance_sampling_ratio/max": 1.2573319673538208,
      "sampling/importance_sampling_ratio/mean": 1.0008736848831177,
      "sampling/importance_sampling_ratio/min": 0.8466115593910217,
      "sampling/sampling_logp_difference/max": 0.15867352485656738,
      "sampling/sampling_logp_difference/mean": 0.011756965890526772,
      "step": 9,
      "step_time": 12.814324255000429
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 0.46368078887462616,
      "epoch": 0.0002,
      "grad_norm": 0.4458825886249542,
      "kl": 0.0005188173963688314,
      "learning_rate": 7.714285714285716e-07,
      "loss": 0.0428,
      "step": 10,
      "step_time": 3.553372275999209
    },
    {
      "epoch": 0.0002,
      "eval_clip_ratio/high_max": 0.0,
      "eval_clip_ratio/high_mean": 0.0,
      "eval_clip_ratio/low_mean": 0.0,
      "eval_clip_ratio/low_min": 0.0,
      "eval_clip_ratio/region_mean": 0.0,
      "eval_completions/clipped_ratio": 0.7,
      "eval_completions/max_length": 16.0,
      "eval_completions/max_terminated_length": 9.8,
      "eval_completions/mean_length": 14.7,
      "eval_completions/mean_terminated_length": 9.16666669845581,
      "eval_completions/min_length": 11.8,
      "eval_completions/min_terminated_length": 8.6,
      "eval_entropy": 0.40578661262989046,
      "eval_frac_reward_zero_std": 0.0,
      "eval_kl": 0.00042531737562967464,
      "eval_loss": 0.02340291067957878,
      "eval_num_tokens": 262651.0,
      "eval_reward": -18.063774490356444,
      "eval_reward_std": 14.382386589050293,
      "eval_rewards/rollout_reward_func/mean": -18.063774490356444,
      "eval_rewards/rollout_reward_func/std": 14.382386589050293,
      "eval_runtime": 13.98,
      "eval_samples_per_second": 0.715,
      "eval_sampling/importance_sampling_ratio/max": 1.1354158759117126,
      "eval_sampling/importance_sampling_ratio/mean": 1.0028302788734436,
      "eval_sampling/importance_sampling_ratio/min": 0.9048627257347107,
      "eval_sampling/sampling_logp_difference/max": 0.13887375593185425,
      "eval_sampling/sampling_logp_difference/mean": 0.011902038706466556,
      "eval_steps_per_second": 0.215,
      "step": 10
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 100000,
  "num_input_tokens_seen": 262651,
  "num_train_epochs": 2,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}