{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.07011217948717949,
  "eval_steps": 50,
  "global_step": 350,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0004,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 40.58326721191406,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.0,
      "num_tokens": 8880.0,
      "reward": 0.8125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 1
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0008,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 41.42949676513672,
      "kl": 0.0,
      "learning_rate": 4e-08,
      "loss": 0.0,
      "num_tokens": 17760.0,
      "reward": 0.25,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 2
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0012,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 50.087196350097656,
      "kl": 0.0009548700036248192,
      "learning_rate": 8e-08,
      "loss": 0.0,
      "num_tokens": 26640.0,
      "reward": 0.25,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 3
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0016,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 34.29154586791992,
      "kl": 0.0010083172237500548,
      "learning_rate": 1.2e-07,
      "loss": 0.0,
      "num_tokens": 35520.0,
      "reward": 0.25,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 4
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.002,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 52.334266662597656,
      "kl": 0.0011691565159708261,
      "learning_rate": 1.6e-07,
      "loss": 0.0,
      "num_tokens": 44400.0,
      "reward": 0.75,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 5
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0024,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 46.0760612487793,
      "kl": 0.001076721484423615,
      "learning_rate": 2e-07,
      "loss": 0.0,
      "num_tokens": 53280.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 6
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0028,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 43.4420280456543,
      "kl": 0.0012002449948340654,
      "learning_rate": 2.4e-07,
      "loss": 0.0,
      "num_tokens": 62160.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 7
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0032,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 28.63946533203125,
      "kl": 0.0021694210590794683,
      "learning_rate": 2.8e-07,
      "loss": 0.0,
      "num_tokens": 71040.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 8
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0036,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 26.870580673217773,
      "kl": 0.0032091100583784282,
      "learning_rate": 3.2e-07,
      "loss": 0.0,
      "num_tokens": 79920.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 9
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.004,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 26.891817092895508,
      "kl": 0.0060052815824747086,
      "learning_rate": 3.6e-07,
      "loss": 0.0,
      "num_tokens": 88800.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 10
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0044,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 25.42474365234375,
      "kl": 0.011448211967945099,
      "learning_rate": 4e-07,
      "loss": 0.0,
      "num_tokens": 97680.0,
      "reward": 0.125,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 11
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0048,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 44.18297576904297,
      "kl": 0.01896629249677062,
      "learning_rate": 4.3999999999999997e-07,
      "loss": 0.0001,
      "num_tokens": 106560.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 12
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0052,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 44.807559967041016,
      "kl": 0.02193804900161922,
      "learning_rate": 4.8e-07,
      "loss": 0.0001,
      "num_tokens": 115440.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 13
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0056,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 33.35441970825195,
      "kl": 0.021487060468643904,
      "learning_rate": 5.2e-07,
      "loss": 0.0001,
      "num_tokens": 124320.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 14
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.006,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 39.03325653076172,
      "kl": 0.03645531600341201,
      "learning_rate": 5.6e-07,
      "loss": 0.0001,
      "num_tokens": 133200.0,
      "reward": 0.5,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 15
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0064,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 39.2026481628418,
      "kl": 0.041529999114573,
      "learning_rate": 6e-07,
      "loss": 0.0002,
      "num_tokens": 142080.0,
      "reward": 0.375,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 16
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0068,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1584884077310562,
      "kl": 0.05437520984560251,
      "learning_rate": 6.4e-07,
      "loss": 0.0002,
      "num_tokens": 150960.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 17
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0072,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.19170306622982025,
      "kl": 0.050666666589677334,
      "learning_rate": 6.800000000000001e-07,
      "loss": 0.0002,
      "num_tokens": 159840.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 18
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0076,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 45.9601936340332,
      "kl": 0.05384485796093941,
      "learning_rate": 7.2e-07,
      "loss": 0.0002,
      "num_tokens": 168720.0,
      "reward": 0.3125,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 19
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.008,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 34.677940368652344,
      "kl": 0.054538323543965816,
      "learning_rate": 7.599999999999999e-07,
      "loss": 0.0002,
      "num_tokens": 177600.0,
      "reward": 0.875,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 20
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0084,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 35.03508758544922,
      "kl": 0.05612980853766203,
      "learning_rate": 8e-07,
      "loss": 0.0002,
      "num_tokens": 186480.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 21
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0088,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 24.550140380859375,
      "kl": 0.06290358863770962,
      "learning_rate": 8.399999999999999e-07,
      "loss": 0.0003,
      "num_tokens": 195360.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 22
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0092,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 28.05525779724121,
      "kl": 0.06588808912783861,
      "learning_rate": 8.799999999999999e-07,
      "loss": 0.0003,
      "num_tokens": 204240.0,
      "reward": 0.1875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 23
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0096,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 40.67657470703125,
      "kl": 0.08455395698547363,
      "learning_rate": 9.2e-07,
      "loss": 0.0003,
      "num_tokens": 213120.0,
      "reward": 0.75,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 24
    },
    {
      "completion_length": 504.5,
      "completions/clipped_ratio": 0.875,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 452.0,
      "completions/mean_length": 504.5,
      "completions/mean_terminated_length": 452.0,
      "completions/min_length": 452.0,
      "completions/min_terminated_length": 452.0,
      "epoch": 0.01,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 30.287317276000977,
      "kl": 0.08439960144460201,
      "learning_rate": 9.6e-07,
      "loss": 0.0003,
      "num_tokens": 221880.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 25
    },
    {
      "completion_length": 488.0,
      "completions/clipped_ratio": 0.9375,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 128.0,
      "completions/mean_length": 488.0,
      "completions/mean_terminated_length": 128.0,
      "completions/min_length": 128.0,
      "completions/min_terminated_length": 128.0,
      "epoch": 0.0104,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 59.74070739746094,
      "kl": 0.10757334344089031,
      "learning_rate": 1e-06,
      "loss": 0.0004,
      "num_tokens": 230376.0,
      "reward": 0.375,
      "reward_std": 0.5,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 26
    },
    {
      "completion_length": 511.25,
      "completions/clipped_ratio": 0.875,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 508.0,
      "completions/mean_length": 511.25,
      "completions/mean_terminated_length": 506.0,
      "completions/min_length": 504.0,
      "completions/min_terminated_length": 504.0,
      "epoch": 0.0108,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 42.04596710205078,
      "kl": 0.10228226706385612,
      "learning_rate": 1.04e-06,
      "loss": 0.0004,
      "num_tokens": 239244.0,
      "reward": 0.3125,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 27
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0112,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 20.101566314697266,
      "kl": 0.09485878236591816,
      "learning_rate": 1.08e-06,
      "loss": 0.0004,
      "num_tokens": 248124.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 28
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0116,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 31.395845413208008,
      "kl": 0.1172705814242363,
      "learning_rate": 1.12e-06,
      "loss": 0.0005,
      "num_tokens": 257004.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 29
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.012,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 32.50520706176758,
      "kl": 0.10939355939626694,
      "learning_rate": 1.16e-06,
      "loss": 0.0004,
      "num_tokens": 265884.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 30
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0124,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 23.288822174072266,
      "kl": 0.11976311914622784,
      "learning_rate": 1.2e-06,
      "loss": 0.0005,
      "num_tokens": 274764.0,
      "reward": 0.125,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 31
    },
    {
      "completion_length": 507.5625,
      "completions/clipped_ratio": 0.9375,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 441.0,
      "completions/mean_length": 507.5625,
      "completions/mean_terminated_length": 441.0,
      "completions/min_length": 441.0,
      "completions/min_terminated_length": 441.0,
      "epoch": 0.0128,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 17.808048248291016,
      "kl": 0.12108245491981506,
      "learning_rate": 1.24e-06,
      "loss": 0.0005,
      "num_tokens": 283573.0,
      "reward": 0.6875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 32
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0132,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 38.9638557434082,
      "kl": 0.13311625644564629,
      "learning_rate": 1.28e-06,
      "loss": 0.0005,
      "num_tokens": 292453.0,
      "reward": 0.75,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 33
    },
    {
      "completion_length": 511.9375,
      "completions/clipped_ratio": 0.9375,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 511.0,
      "completions/mean_length": 511.9375,
      "completions/mean_terminated_length": 511.0,
      "completions/min_length": 511.0,
      "completions/min_terminated_length": 511.0,
      "epoch": 0.0136,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 35.36914825439453,
      "kl": 0.15050986409187317,
      "learning_rate": 1.32e-06,
      "loss": 0.0006,
      "num_tokens": 301332.0,
      "reward": 0.5,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 34
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.014,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.13711945712566376,
      "kl": 0.1605079583823681,
      "learning_rate": 1.3600000000000001e-06,
      "loss": 0.0006,
      "num_tokens": 310212.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 35
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0144,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.1120786964893341,
      "kl": 0.1808343380689621,
      "learning_rate": 1.4e-06,
      "loss": 0.0007,
      "num_tokens": 319092.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 36
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0148,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 25.245079040527344,
      "kl": 0.17078161984682083,
      "learning_rate": 1.44e-06,
      "loss": 0.0007,
      "num_tokens": 327972.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 37
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0152,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 37.300872802734375,
      "kl": 0.17071689665317535,
      "learning_rate": 1.48e-06,
      "loss": 0.0007,
      "num_tokens": 336852.0,
      "reward": 0.25,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 38
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0156,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 34.02109909057617,
      "kl": 0.17260625213384628,
      "learning_rate": 1.5199999999999998e-06,
      "loss": 0.0007,
      "num_tokens": 345732.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 39
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.016,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 22.01497459411621,
      "kl": 0.15412141010165215,
      "learning_rate": 1.5599999999999999e-06,
      "loss": 0.0006,
      "num_tokens": 354612.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 40
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0164,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 34.2176399230957,
      "kl": 0.18766384199261665,
      "learning_rate": 1.6e-06,
      "loss": 0.0008,
      "num_tokens": 363492.0,
      "reward": 0.5,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 41
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0168,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3294.3701171875,
      "kl": 7.936608750373125,
      "learning_rate": 1.6399999999999998e-06,
      "loss": 0.0317,
      "num_tokens": 372372.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 42
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0172,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 33.57102966308594,
      "kl": 0.19400563091039658,
      "learning_rate": 1.6799999999999998e-06,
      "loss": 0.0008,
      "num_tokens": 381252.0,
      "reward": 0.125,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 43
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0176,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 29.105810165405273,
      "kl": 0.17841431871056557,
      "learning_rate": 1.7199999999999998e-06,
      "loss": 0.0007,
      "num_tokens": 390132.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 44
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.018,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 31.889549255371094,
      "kl": 0.16708053275942802,
      "learning_rate": 1.7599999999999999e-06,
      "loss": 0.0007,
      "num_tokens": 399012.0,
      "reward": 0.25,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 45
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0184,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 27.441055297851562,
      "kl": 0.1539367213845253,
      "learning_rate": 1.8e-06,
      "loss": 0.0006,
      "num_tokens": 407892.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 46
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0188,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 25.701475143432617,
      "kl": 0.17651546001434326,
      "learning_rate": 1.84e-06,
      "loss": 0.0007,
      "num_tokens": 416772.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 47
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0192,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 17.01966667175293,
      "kl": 0.16722217947244644,
      "learning_rate": 1.8799999999999998e-06,
      "loss": 0.0007,
      "num_tokens": 425652.0,
      "reward": 0.1875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 48
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0196,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09812258183956146,
      "kl": 0.20355528965592384,
      "learning_rate": 1.92e-06,
      "loss": 0.0008,
      "num_tokens": 434532.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 49
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.02,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 24.149648666381836,
      "kl": 0.2126757726073265,
      "learning_rate": 1.96e-06,
      "loss": 0.0009,
      "num_tokens": 443412.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 50
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.02,
      "eval_loss": 0.0002288773684995249,
      "eval_runtime": 12487.3227,
      "eval_samples_per_second": 0.016,
      "eval_steps_per_second": 0.004,
      "frac_reward_zero_std": 0.58,
      "num_tokens": 554412.0,
      "reward": 0.415,
      "reward_std": 0.21928203105926514,
      "rewards/compute_reward/mean": 0.415,
      "rewards/compute_reward/std": 0.21928203105926514,
      "step": 50
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0204,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10443887859582901,
      "kl": 0.2396080531179905,
      "learning_rate": 2e-06,
      "loss": 0.001,
      "num_tokens": 563292.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 51
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0208,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.07596234977245331,
      "kl": 0.21716779470443726,
      "learning_rate": 1.9999756307053944e-06,
      "loss": 0.0009,
      "num_tokens": 572172.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 52
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0212,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.932524681091309,
      "kl": 0.22818409651517868,
      "learning_rate": 1.999902524009304e-06,
      "loss": 0.2509,
      "num_tokens": 581052.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 53
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0216,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 17.921937942504883,
      "kl": 0.2106623314321041,
      "learning_rate": 1.999780683474845e-06,
      "loss": -0.4991,
      "num_tokens": 589932.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 54
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.022,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 8.580633163452148,
      "kl": 0.24051300063729286,
      "learning_rate": 1.999610115040354e-06,
      "loss": 0.2509,
      "num_tokens": 598812.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 55
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0224,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 23.697641372680664,
      "kl": 0.2225131280720234,
      "learning_rate": 1.9993908270190957e-06,
      "loss": 0.2174,
      "num_tokens": 607692.0,
      "reward": 0.25,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 56
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0228,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 23.44224739074707,
      "kl": 0.21955753862857819,
      "learning_rate": 1.999122830098858e-06,
      "loss": -0.6485,
      "num_tokens": 616572.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 57
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0232,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 18.962669372558594,
      "kl": 0.28268108516931534,
      "learning_rate": 1.998806137341434e-06,
      "loss": 0.4341,
      "num_tokens": 625452.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 58
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0236,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 32.13456726074219,
      "kl": 0.24665188044309616,
      "learning_rate": 1.998440764181981e-06,
      "loss": 0.001,
      "num_tokens": 634332.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 59
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.024,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 27.285778045654297,
      "kl": 0.25285593420267105,
      "learning_rate": 1.9980267284282714e-06,
      "loss": -0.3739,
      "num_tokens": 643212.0,
      "reward": 0.1875,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 60
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0244,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 18.179141998291016,
      "kl": 0.26473674178123474,
      "learning_rate": 1.997564050259824e-06,
      "loss": 0.251,
      "num_tokens": 652092.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 61
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0248,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 23.763172149658203,
      "kl": 0.3116937354207039,
      "learning_rate": 1.99705275222692e-06,
      "loss": 0.7511,
      "num_tokens": 660972.0,
      "reward": 0.6875,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 62
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0252,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 27.499988555908203,
      "kl": 0.32043570652604103,
      "learning_rate": 1.9964928592495045e-06,
      "loss": -0.9985,
      "num_tokens": 669852.0,
      "reward": 0.75,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 63
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0256,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 22.124732971191406,
      "kl": 0.3462621197104454,
      "learning_rate": 1.99588439861597e-06,
      "loss": -0.0566,
      "num_tokens": 678732.0,
      "reward": 0.75,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 64
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.026,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 32.59796905517578,
      "kl": 0.38544727861881256,
      "learning_rate": 1.995227399981831e-06,
      "loss": 1.0594,
      "num_tokens": 687612.0,
      "reward": 0.5,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 65
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0264,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 41.3852424621582,
      "kl": 0.4055018573999405,
      "learning_rate": 1.994521895368273e-06,
      "loss": -1.1232,
      "num_tokens": 696492.0,
      "reward": 0.5625,
      "reward_std": 0.51933753490448,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 66
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0268,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 28.504070281982422,
      "kl": 0.4306362494826317,
      "learning_rate": 1.9937679191605962e-06,
      "loss": 0.3766,
      "num_tokens": 705372.0,
      "reward": 0.1875,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 67
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0272,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 33.86847686767578,
      "kl": 0.4562436640262604,
      "learning_rate": 1.992965508106537e-06,
      "loss": -0.2146,
      "num_tokens": 714252.0,
      "reward": 0.6875,
      "reward_std": 0.51933753490448,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 68
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0276,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 30.6969051361084,
      "kl": 0.4892871007323265,
      "learning_rate": 1.9921147013144777e-06,
      "loss": 0.7183,
      "num_tokens": 723132.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 69
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.028,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 16.82533836364746,
      "kl": 0.4661508873105049,
      "learning_rate": 1.9912155402515414e-06,
      "loss": -0.3731,
      "num_tokens": 732012.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 70
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0284,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 16.892255783081055,
      "kl": 0.5023886859416962,
      "learning_rate": 1.99026806874157e-06,
      "loss": 0.002,
      "num_tokens": 740892.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 71
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0288,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 28.076086044311523,
      "kl": 0.4288725033402443,
      "learning_rate": 1.9892723329629885e-06,
      "loss": 0.7181,
      "num_tokens": 749772.0,
      "reward": 0.625,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 72
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0292,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 26.868600845336914,
      "kl": 0.43238623440265656,
      "learning_rate": 1.9882283814465526e-06,
      "loss": -0.9311,
      "num_tokens": 758652.0,
      "reward": 0.4375,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 73
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0296,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 22.524154663085938,
      "kl": 0.4475055858492851,
      "learning_rate": 1.987136265072988e-06,
      "loss": 0.2183,
      "num_tokens": 767532.0,
      "reward": 0.5,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 74
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.03,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 26.058460235595703,
      "kl": 0.4978598430752754,
      "learning_rate": 1.985996037070505e-06,
      "loss": -0.123,
      "num_tokens": 776412.0,
      "reward": 0.4375,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 75
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0304,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 25.16863250732422,
      "kl": 0.39492419362068176,
      "learning_rate": 1.984807753012208e-06,
      "loss": 0.2515,
      "num_tokens": 785292.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 76
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0308,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 26.64851951599121,
      "kl": 0.37667443603277206,
      "learning_rate": 1.983571470813386e-06,
      "loss": -0.3734,
      "num_tokens": 794172.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 77
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0312,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 23.185409545898438,
      "kl": 0.38686759769916534,
      "learning_rate": 1.9822872507286887e-06,
      "loss": 0.2515,
      "num_tokens": 803052.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 78
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0316,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.08748236298561096,
      "kl": 0.4463553726673126,
      "learning_rate": 1.9809551553491913e-06,
      "loss": 0.0018,
      "num_tokens": 811932.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 79
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.032,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 24.110332489013672,
      "kl": 0.36003363132476807,
      "learning_rate": 1.979575249599344e-06,
      "loss": -0.215,
      "num_tokens": 820812.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 80
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0324,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 18.954856872558594,
      "kl": 0.36876556277275085,
      "learning_rate": 1.9781476007338054e-06,
      "loss": 0.093,
      "num_tokens": 829692.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 81
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0328,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.058808326721191,
      "kl": 0.403683140873909,
      "learning_rate": 1.9766722783341677e-06,
      "loss": -0.2483,
      "num_tokens": 838572.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 82
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0332,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 23.458297729492188,
      "kl": 0.4015289470553398,
      "learning_rate": 1.975149354305563e-06,
      "loss": 0.6265,
      "num_tokens": 847452.0,
      "reward": 0.4375,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 83
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0336,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 21.95713996887207,
      "kl": 0.3268972262740135,
      "learning_rate": 1.97357890287316e-06,
      "loss": -0.2486,
      "num_tokens": 856332.0,
      "reward": 0.25,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 84
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.034,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 28.32386589050293,
      "kl": 0.38899460434913635,
      "learning_rate": 1.9719610005785463e-06,
      "loss": -0.7148,
      "num_tokens": 865212.0,
      "reward": 0.625,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 85
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0344,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 41.013309478759766,
      "kl": 0.5532749891281128,
      "learning_rate": 1.9702957262759963e-06,
      "loss": 0.7186,
      "num_tokens": 874092.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 86
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0348,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 49.202369689941406,
      "kl": 0.5244565531611443,
      "learning_rate": 1.968583161128631e-06,
      "loss": 0.377,
      "num_tokens": 882972.0,
      "reward": 0.4375,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 87
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0352,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 456.4792785644531,
      "kl": 3.696738824248314,
      "learning_rate": 1.9668233886044593e-06,
      "loss": -0.0187,
      "num_tokens": 891852.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 88
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0356,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 77.5242919921875,
      "kl": 0.61634162068367,
      "learning_rate": 1.9650164944723112e-06,
      "loss": 0.0025,
      "num_tokens": 900732.0,
      "reward": 0.4375,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 89
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.036,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 32.868568420410156,
      "kl": 0.5600514858961105,
      "learning_rate": 1.963162566797658e-06,
      "loss": -0.3392,
      "num_tokens": 909612.0,
      "reward": 0.5,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 90
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0364,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 32.02371597290039,
      "kl": 0.5822872817516327,
      "learning_rate": 1.9612616959383188e-06,
      "loss": -0.3391,
      "num_tokens": 918492.0,
      "reward": 0.6875,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 91
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0368,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 44.31180953979492,
      "kl": 0.7215879261493683,
      "learning_rate": 1.9593139745400573e-06,
      "loss": 0.8442,
      "num_tokens": 927372.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 92
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0372,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 61.617347717285156,
      "kl": 0.5873569175601006,
      "learning_rate": 1.957319497532067e-06,
      "loss": -0.7475,
      "num_tokens": 936252.0,
      "reward": 0.75,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 93
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0376,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 45.146324157714844,
      "kl": 0.5552276074886322,
      "learning_rate": 1.9552783621223435e-06,
      "loss": 0.1272,
      "num_tokens": 945132.0,
      "reward": 0.8125,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 94
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.038,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 24.78557586669922,
      "kl": 0.5455406308174133,
      "learning_rate": 1.953190667792947e-06,
      "loss": 0.1272,
      "num_tokens": 954012.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 95
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0384,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.901802062988281,
      "kl": 0.6519858688116074,
      "learning_rate": 1.9510565162951534e-06,
      "loss": -0.3723,
      "num_tokens": 962892.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 96
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0388,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 34.490142822265625,
      "kl": 0.6270407885313034,
      "learning_rate": 1.9488760116444964e-06,
      "loss": 0.6274,
      "num_tokens": 971772.0,
      "reward": 0.5625,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 97
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0392,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 21.449689865112305,
      "kl": 0.6352937817573547,
      "learning_rate": 1.9466492601156963e-06,
      "loss": -0.2474,
      "num_tokens": 980652.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 98
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0396,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 26.181488037109375,
      "kl": 0.6954683512449265,
      "learning_rate": 1.944376370237481e-06,
      "loss": -0.2472,
      "num_tokens": 989532.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 99
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 43.50558853149414,
      "kl": 0.6294843703508377,
      "learning_rate": 1.9420574527872966e-06,
      "loss": 0.6274,
      "num_tokens": 998412.0,
      "reward": 0.25,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 100
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04,
      "eval_loss": 0.13216377794742584,
      "eval_runtime": 13737.4228,
      "eval_samples_per_second": 0.015,
      "eval_steps_per_second": 0.004,
      "frac_reward_zero_std": 0.62,
      "num_tokens": 1109412.0,
      "reward": 0.43,
      "reward_std": 0.20082903623580933,
      "rewards/compute_reward/mean": 0.43,
      "rewards/compute_reward/std": 0.20082903623580933,
      "step": 100
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0404,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.638951301574707,
      "kl": 0.7218647003173828,
      "learning_rate": 1.9396926207859082e-06,
      "loss": 0.2528,
      "num_tokens": 1118292.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 101
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0408,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 38.83489227294922,
      "kl": 0.7351299375295639,
      "learning_rate": 1.9372819894918914e-06,
      "loss": -0.497,
      "num_tokens": 1127172.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 102
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0412,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 45.73721694946289,
      "kl": 0.7570449411869049,
      "learning_rate": 1.9348256763960142e-06,
      "loss": 0.1615,
      "num_tokens": 1136052.0,
      "reward": 0.5625,
      "reward_std": 0.5580127239227295,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 103
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0416,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 33.400882720947266,
      "kl": 0.6374172419309616,
      "learning_rate": 1.9323238012155122e-06,
      "loss": 0.219,
      "num_tokens": 1144932.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 104
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.042,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 27.644853591918945,
      "kl": 0.7628162950277328,
      "learning_rate": 1.929776485888251e-06,
      "loss": -0.2804,
      "num_tokens": 1153812.0,
      "reward": 0.1875,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 105
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0424,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 22.251258850097656,
      "kl": 0.7362856566905975,
      "learning_rate": 1.9271838545667875e-06,
      "loss": -0.2715,
      "num_tokens": 1162692.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 106
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0428,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 15.995705604553223,
      "kl": 0.7166048735380173,
      "learning_rate": 1.9245460336123133e-06,
      "loss": -0.0306,
      "num_tokens": 1171572.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 107
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0432,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1572.4730224609375,
      "kl": 16.452498733997345,
      "learning_rate": 1.9218631515885003e-06,
      "loss": 0.8737,
      "num_tokens": 1180452.0,
      "reward": 0.1875,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 108
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0436,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 176.10447692871094,
      "kl": 2.2758754789829254,
      "learning_rate": 1.9191353392552343e-06,
      "loss": -0.5823,
      "num_tokens": 1189332.0,
      "reward": 0.1875,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 109
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.044,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 22.86349105834961,
      "kl": 0.695338636636734,
      "learning_rate": 1.9163627295622395e-06,
      "loss": 0.1278,
      "num_tokens": 1198212.0,
      "reward": 0.0625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 110
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0444,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 23.30611801147461,
      "kl": 0.7727901041507721,
      "learning_rate": 1.9135454576426007e-06,
      "loss": 0.7529,
      "num_tokens": 1207092.0,
      "reward": 0.375,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 111
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0448,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 32.70335006713867,
      "kl": 0.7462340593338013,
      "learning_rate": 1.910683660806177e-06,
      "loss": -0.8384,
      "num_tokens": 1215972.0,
      "reward": 0.5,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 112
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0452,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 81.09750366210938,
      "kl": 1.7529860734939575,
      "learning_rate": 1.9077774785329087e-06,
      "loss": 0.5314,
      "num_tokens": 1224852.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 113
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0456,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 30.61618423461914,
      "kl": 0.7314885407686234,
      "learning_rate": 1.9048270524660196e-06,
      "loss": -0.5215,
      "num_tokens": 1233732.0,
      "reward": 0.125,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 114
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.046,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 25.569091796875,
      "kl": 0.7862952053546906,
      "learning_rate": 1.9018325264051138e-06,
      "loss": 0.2196,
      "num_tokens": 1242612.0,
      "reward": 0.6875,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 115
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0464,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 33.22822952270508,
      "kl": 0.7910800129175186,
      "learning_rate": 1.8987940462991669e-06,
      "loss": -0.4967,
      "num_tokens": 1251492.0,
      "reward": 0.75,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 116
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0468,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 141293649920.0,
      "kl": 182104736.6479945,
      "learning_rate": 1.8957117602394128e-06,
      "loss": 728419.0625,
      "num_tokens": 1260372.0,
      "reward": 0.4375,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 117
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0472,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 19.006919860839844,
      "kl": 0.8009488135576248,
      "learning_rate": 1.8925858184521255e-06,
      "loss": 0.5946,
      "num_tokens": 1269252.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 118
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0476,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 27.448341369628906,
      "kl": 0.8013195097446442,
      "learning_rate": 1.8894163732912974e-06,
      "loss": -0.1218,
      "num_tokens": 1278132.0,
      "reward": 0.125,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 119
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.048,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.09504876285791397,
      "kl": 0.8626277446746826,
      "learning_rate": 1.8862035792312146e-06,
      "loss": 0.0035,
      "num_tokens": 1287012.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 120
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0484,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.10749489068984985,
      "kl": 0.8514651656150818,
      "learning_rate": 1.8829475928589268e-06,
      "loss": 0.0034,
      "num_tokens": 1295892.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 121
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0488,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 24.188209533691406,
      "kl": 0.8452038317918777,
      "learning_rate": 1.8796485728666165e-06,
      "loss": -0.771,
      "num_tokens": 1304772.0,
      "reward": 0.6875,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 122
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0492,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 34.62675476074219,
      "kl": 0.8078770935535431,
      "learning_rate": 1.8763066800438634e-06,
      "loss": 0.5277,
      "num_tokens": 1313652.0,
      "reward": 0.625,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 123
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0496,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 29.03681182861328,
      "kl": 0.8311980664730072,
      "learning_rate": 1.8729220772698095e-06,
      "loss": 0.1283,
      "num_tokens": 1322532.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 124
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 19.019657135009766,
      "kl": 0.7919216603040695,
      "learning_rate": 1.869494929505219e-06,
      "loss": 0.0032,
      "num_tokens": 1331412.0,
      "reward": 0.0625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 125
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0504,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 15.407096862792969,
      "kl": 0.7951618134975433,
      "learning_rate": 1.8660254037844386e-06,
      "loss": 0.1282,
      "num_tokens": 1340292.0,
      "reward": 0.625,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 126
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0508,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 17.479686737060547,
      "kl": 0.8314408957958221,
      "learning_rate": 1.8625136692072574e-06,
      "loss": -0.2466,
      "num_tokens": 1349172.0,
      "reward": 0.6875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 127
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0512,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 29.629783630371094,
      "kl": 0.8269994258880615,
      "learning_rate": 1.8589598969306644e-06,
      "loss": 0.8112,
      "num_tokens": 1358052.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 128
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0516,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 16.603778839111328,
      "kl": 0.8076073229312897,
      "learning_rate": 1.8553642601605066e-06,
      "loss": -0.5547,
      "num_tokens": 1366932.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 129
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.052,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 21.11556625366211,
      "kl": 0.7634301334619522,
      "learning_rate": 1.8517269341430474e-06,
      "loss": 0.253,
      "num_tokens": 1375812.0,
      "reward": 0.3125,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 130
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0524,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 25.48746109008789,
      "kl": 0.840743288397789,
      "learning_rate": 1.8480480961564257e-06,
      "loss": -0.2466,
      "num_tokens": 1384692.0,
      "reward": 0.875,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 131
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0528,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 15.836709022521973,
      "kl": 0.8535377830266953,
      "learning_rate": 1.844327925502015e-06,
      "loss": 0.0034,
      "num_tokens": 1393572.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 132
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0532,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 27.47414207458496,
      "kl": 0.7292637825012207,
      "learning_rate": 1.8405666034956842e-06,
      "loss": -0.1221,
      "num_tokens": 1402452.0,
      "reward": 0.5,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 133
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0536,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 24.72193717956543,
      "kl": 0.7159179598093033,
      "learning_rate": 1.8367643134589616e-06,
      "loss": 0.0029,
      "num_tokens": 1411332.0,
      "reward": 0.4375,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 134
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.054,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 72.50166320800781,
      "kl": 0.7516931444406509,
      "learning_rate": 1.8329212407100993e-06,
      "loss": -0.2469,
      "num_tokens": 1420212.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 135
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0544,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 13.944753646850586,
      "kl": 0.824699729681015,
      "learning_rate": 1.8290375725550415e-06,
      "loss": 0.3782,
      "num_tokens": 1429092.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 136
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0548,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 17.254592895507812,
      "kl": 0.7634341418743134,
      "learning_rate": 1.825113498278295e-06,
      "loss": 0.6279,
      "num_tokens": 1437972.0,
      "reward": 0.5,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 137
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0552,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 43.185054779052734,
      "kl": 0.831812858581543,
      "learning_rate": 1.821149209133704e-06,
      "loss": -1.1214,
      "num_tokens": 1446852.0,
      "reward": 0.3125,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 138
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0556,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 26.895647048950195,
      "kl": 0.7653562426567078,
      "learning_rate": 1.8171448983351283e-06,
      "loss": 0.2285,
      "num_tokens": 1455732.0,
      "reward": 0.3125,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 139
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.056,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 31.948415756225586,
      "kl": 0.8429819345474243,
      "learning_rate": 1.8131007610470274e-06,
      "loss": 0.2779,
      "num_tokens": 1464612.0,
      "reward": 0.75,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 140
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0564,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 16.335346221923828,
      "kl": 0.7552044689655304,
      "learning_rate": 1.8090169943749474e-06,
      "loss": -0.122,
      "num_tokens": 1473492.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 141
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0568,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 13.47604751586914,
      "kl": 0.728036180138588,
      "learning_rate": 1.804893797355914e-06,
      "loss": 0.1279,
      "num_tokens": 1482372.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 142
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0572,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 17.503402709960938,
      "kl": 0.7506053000688553,
      "learning_rate": 1.8007313709487333e-06,
      "loss": 0.003,
      "num_tokens": 1491252.0,
      "reward": 0.25,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 143
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0576,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 27.46441078186035,
      "kl": 0.8133435249328613,
      "learning_rate": 1.7965299180241961e-06,
      "loss": -0.0882,
      "num_tokens": 1500132.0,
      "reward": 0.625,
      "reward_std": 0.5386751294136047,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 144
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.058,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 31.184906005859375,
      "kl": 0.7688865214586258,
      "learning_rate": 1.7922896433551906e-06,
      "loss": 0.5945,
      "num_tokens": 1509012.0,
      "reward": 0.25,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 145
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0584,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 24.311113357543945,
      "kl": 0.8520800322294235,
      "learning_rate": 1.7880107536067217e-06,
      "loss": -0.1216,
      "num_tokens": 1517892.0,
      "reward": 0.3125,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 146
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0588,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 24.806241989135742,
      "kl": 0.7743303179740906,
      "learning_rate": 1.7836934573258397e-06,
      "loss": -0.7468,
      "num_tokens": 1526772.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 147
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0592,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 29.297000885009766,
      "kl": 0.7643304467201233,
      "learning_rate": 1.7793379649314742e-06,
      "loss": 0.7864,
      "num_tokens": 1535652.0,
      "reward": 0.4375,
      "reward_std": 0.51933753490448,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 148
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0596,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 30.811779022216797,
      "kl": 0.8101753890514374,
      "learning_rate": 1.7749444887041795e-06,
      "loss": -0.2222,
      "num_tokens": 1544532.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 149
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 34.043434143066406,
      "kl": 0.7941624373197556,
      "learning_rate": 1.7705132427757892e-06,
      "loss": -0.8627,
      "num_tokens": 1553412.0,
      "reward": 0.4375,
      "reward_std": 0.5580127239227295,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 150
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06,
      "eval_loss": 0.006696930155158043,
      "eval_runtime": 8120.1752,
      "eval_samples_per_second": 0.025,
      "eval_steps_per_second": 0.006,
      "frac_reward_zero_std": 0.62,
      "num_tokens": 1664412.0,
      "reward": 0.475,
      "reward_std": 0.19618802070617675,
      "rewards/compute_reward/mean": 0.475,
      "rewards/compute_reward/std": 0.19618802070617675,
      "step": 150
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0604,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 62.78795623779297,
      "kl": 0.8813759088516235,
      "learning_rate": 1.766044443118978e-06,
      "loss": -0.3714,
      "num_tokens": 1673292.0,
      "reward": 0.0625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 151
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0608,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 16.987716674804688,
      "kl": 0.7363490760326385,
      "learning_rate": 1.7615383075367368e-06,
      "loss": -0.43,
      "num_tokens": 1682172.0,
      "reward": 0.1875,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 152
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0612,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 17.806154251098633,
      "kl": 0.8252189606428146,
      "learning_rate": 1.7569950556517563e-06,
      "loss": 0.4362,
      "num_tokens": 1691052.0,
      "reward": 0.125,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 153
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0616,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 23.90774917602539,
      "kl": 0.6684602424502373,
      "learning_rate": 1.7524149088957242e-06,
      "loss": -0.3723,
      "num_tokens": 1699932.0,
      "reward": 0.5625,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 154
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.062,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 19.944137573242188,
      "kl": 0.6719812899827957,
      "learning_rate": 1.747798090498532e-06,
      "loss": 0.8775,
      "num_tokens": 1708812.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 155
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0624,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 13.792506217956543,
      "kl": 0.6391888558864594,
      "learning_rate": 1.743144825477394e-06,
      "loss": -0.6223,
      "num_tokens": 1717692.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 156
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0628,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 5.927950859069824,
      "kl": 0.5306425616145134,
      "learning_rate": 1.738455340625884e-06,
      "loss": 0.0021,
      "num_tokens": 1726572.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 157
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0632,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.993011474609375,
      "kl": 0.5052112117409706,
      "learning_rate": 1.7337298645028762e-06,
      "loss": 0.252,
      "num_tokens": 1735452.0,
      "reward": 0.375,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 158
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0636,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 11.660005569458008,
      "kl": 0.4188714995980263,
      "learning_rate": 1.7289686274214115e-06,
      "loss": -0.3732,
      "num_tokens": 1744332.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 159
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.064,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 18.78450584411621,
      "kl": 0.428101621568203,
      "learning_rate": 1.7241718614374676e-06,
      "loss": 0.3766,
      "num_tokens": 1753212.0,
      "reward": 0.5,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 160
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0644,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 11.88436508178711,
      "kl": 0.3962597921490669,
      "learning_rate": 1.719339800338651e-06,
      "loss": -0.2484,
      "num_tokens": 1762092.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 161
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0648,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04197113960981369,
      "kl": 0.43298500031232834,
      "learning_rate": 1.7144726796328032e-06,
      "loss": 0.0017,
      "num_tokens": 1770972.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 162
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0652,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 6.9180450439453125,
      "kl": 0.37740715220570564,
      "learning_rate": 1.7095707365365209e-06,
      "loss": 0.1265,
      "num_tokens": 1779852.0,
      "reward": 0.3125,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 163
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0656,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 16.20522117614746,
      "kl": 0.37258073687553406,
      "learning_rate": 1.7046342099635947e-06,
      "loss": -0.3734,
      "num_tokens": 1788732.0,
      "reward": 0.375,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 164
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.066,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 13.544259071350098,
      "kl": 0.45895127207040787,
      "learning_rate": 1.6996633405133653e-06,
      "loss": 0.5017,
      "num_tokens": 1797612.0,
      "reward": 0.6875,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 165
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0664,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 14.154873847961426,
      "kl": 0.41937466710805893,
      "learning_rate": 1.6946583704589972e-06,
      "loss": -0.3732,
      "num_tokens": 1806492.0,
      "reward": 0.75,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 166
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0668,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 6.917255878448486,
      "kl": 0.34217433631420135,
      "learning_rate": 1.6896195437356697e-06,
      "loss": 0.1263,
      "num_tokens": 1815372.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 167
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0672,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.267975807189941,
      "kl": 0.3709218502044678,
      "learning_rate": 1.6845471059286886e-06,
      "loss": 0.2514,
      "num_tokens": 1824252.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 168
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0676,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 7.46565055847168,
      "kl": 0.31025706976652145,
      "learning_rate": 1.6794413042615166e-06,
      "loss": -0.2487,
      "num_tokens": 1833132.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 169
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.068,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.313626766204834,
      "kl": 0.33493074029684067,
      "learning_rate": 1.6743023875837233e-06,
      "loss": -0.2151,
      "num_tokens": 1842012.0,
      "reward": 0.25,
      "reward_std": 0.28867512941360474,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 170
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0684,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 11.879616737365723,
      "kl": 0.3535468205809593,
      "learning_rate": 1.669130606358858e-06,
      "loss": 0.2179,
      "num_tokens": 1850892.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 171
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0688,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 6.066073417663574,
      "kl": 0.32792166620492935,
      "learning_rate": 1.6639262126522415e-06,
      "loss": -0.2486,
      "num_tokens": 1859772.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 172
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0692,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 3.136962652206421,
      "kl": 0.370282344520092,
      "learning_rate": 1.6586894601186803e-06,
      "loss": 0.2514,
      "num_tokens": 1868652.0,
      "reward": 0.6875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 173
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0696,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.541350841522217,
      "kl": 0.37312114983797073,
      "learning_rate": 1.6534206039901055e-06,
      "loss": 0.0015,
      "num_tokens": 1877532.0,
      "reward": 0.625,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 174
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.07,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 13.651392936706543,
      "kl": 0.46653878688812256,
      "learning_rate": 1.6481199010631309e-06,
      "loss": 0.2518,
      "num_tokens": 1886412.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 175
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0704,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 12.165966987609863,
      "kl": 0.4945712089538574,
      "learning_rate": 1.6427876096865393e-06,
      "loss": -0.248,
      "num_tokens": 1895292.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 176
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0708,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 16.049345016479492,
      "kl": 0.6074426025152206,
      "learning_rate": 1.6374239897486897e-06,
      "loss": 0.4688,
      "num_tokens": 1904172.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 177
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0712,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 17.58525276184082,
      "kl": 0.4911561757326126,
      "learning_rate": 1.6320293026648508e-06,
      "loss": -1.3393,
      "num_tokens": 1913052.0,
      "reward": 0.3125,
      "reward_std": 0.375,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 178
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0716,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 17.33172035217285,
      "kl": 0.5618030577898026,
      "learning_rate": 1.6266038113644605e-06,
      "loss": 0.7856,
      "num_tokens": 1921932.0,
      "reward": 0.5,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 179
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.072,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 14.58329963684082,
      "kl": 0.5615669637918472,
      "learning_rate": 1.6211477802783102e-06,
      "loss": 0.0937,
      "num_tokens": 1930812.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 180
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0724,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.06436236947774887,
      "kl": 0.583530455827713,
      "learning_rate": 1.615661475325658e-06,
      "loss": 0.0023,
      "num_tokens": 1939692.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 181
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0728,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.032283440232276917,
      "kl": 0.5443942993879318,
      "learning_rate": 1.6101451639012677e-06,
      "loss": 0.0022,
      "num_tokens": 1948572.0,
      "reward": 0.5625,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 182
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0732,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 15.403587341308594,
      "kl": 0.6374123245477676,
      "learning_rate": 1.604599114862375e-06,
      "loss": 0.0025,
      "num_tokens": 1957452.0,
      "reward": 0.4375,
      "reward_std": 0.26933756470680237,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 183
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0736,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 10.842680931091309,
      "kl": 0.5484844744205475,
      "learning_rate": 1.5990235985155857e-06,
      "loss": 0.0022,
      "num_tokens": 1966332.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 184
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.074,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.8211312294006348,
      "kl": 0.6125466376543045,
      "learning_rate": 1.5934188866037015e-06,
      "loss": -0.1225,
      "num_tokens": 1975212.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 185
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0744,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 11.508246421813965,
      "kl": 0.4921167343854904,
      "learning_rate": 1.587785252292473e-06,
      "loss": 0.5934,
      "num_tokens": 1984092.0,
      "reward": 0.75,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 186
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0748,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 10.380646705627441,
      "kl": 0.4468236267566681,
      "learning_rate": 1.5821229701572893e-06,
      "loss": -0.2147,
      "num_tokens": 1992972.0,
      "reward": 0.6875,
      "reward_std": 0.51933753490448,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 187
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0752,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 9.361845016479492,
      "kl": 0.4455399811267853,
      "learning_rate": 1.5764323161697932e-06,
      "loss": 0.1268,
      "num_tokens": 2001852.0,
      "reward": 0.1875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 188
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0756,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 5.388669013977051,
      "kl": 0.4400194138288498,
      "learning_rate": 1.5707135676844319e-06,
      "loss": 0.0018,
      "num_tokens": 2010732.0,
      "reward": 0.4375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 189
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.076,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.730255603790283,
      "kl": 0.45552685111761093,
      "learning_rate": 1.564967003424938e-06,
      "loss": -0.1232,
      "num_tokens": 2019612.0,
      "reward": 0.3125,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 190
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0764,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.63920783996582,
      "kl": 0.5008008778095245,
      "learning_rate": 1.5591929034707466e-06,
      "loss": -0.4979,
      "num_tokens": 2028492.0,
      "reward": 0.625,
      "reward_std": 0.25,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 191
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0768,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 8.034728050231934,
      "kl": 0.47964833676815033,
      "learning_rate": 1.553391549243344e-06,
      "loss": 0.4683,
      "num_tokens": 2037372.0,
      "reward": 0.625,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 192
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0772,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 7.510342121124268,
      "kl": 0.46900178492069244,
      "learning_rate": 1.5475632234925502e-06,
      "loss": 0.1603,
      "num_tokens": 2046252.0,
      "reward": 0.9375,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 193
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0776,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 2.386094570159912,
      "kl": 0.430863693356514,
      "learning_rate": 1.54170821028274e-06,
      "loss": -0.3732,
      "num_tokens": 2055132.0,
      "reward": 0.25,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 194
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.078,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 0.03321576863527298,
      "kl": 0.4494182765483856,
      "learning_rate": 1.5358267949789964e-06,
      "loss": 0.0018,
      "num_tokens": 2064012.0,
      "reward": 0.625,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 195
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0784,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 6.19266414642334,
      "kl": 0.4506584629416466,
      "learning_rate": 1.5299192642332049e-06,
      "loss": -0.5561,
      "num_tokens": 2072892.0,
      "reward": 0.6875,
      "reward_std": 0.41367512941360474,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 196
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0788,
      "frac_reward_zero_std": 0.25,
      "grad_norm": 7.25709867477417,
      "kl": 0.4454643726348877,
      "learning_rate": 1.5239859059700793e-06,
      "loss": 0.3097,
      "num_tokens": 2081772.0,
      "reward": 0.375,
      "reward_std": 0.39433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 197
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0792,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 4.1123151779174805,
      "kl": 0.45052972435951233,
      "learning_rate": 1.5180270093731302e-06,
      "loss": 0.1268,
      "num_tokens": 2090652.0,
      "reward": 0.6875,
      "reward_std": 0.125,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 198
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0796,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 2.5656275749206543,
      "kl": 0.4550231248140335,
      "learning_rate": 1.5120428648705715e-06,
      "loss": 0.1268,
      "num_tokens": 2099532.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 199
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.08,
      "frac_reward_zero_std": 0.75,
      "grad_norm": 1.4234280586242676,
      "kl": 0.44099147617816925,
      "learning_rate": 1.5060337641211636e-06,
      "loss": 0.0018,
      "num_tokens": 2108412.0,
      "reward": 0.375,
      "reward_std": 0.14433756470680237,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 200
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.08,
      "eval_loss": 0.00046315923100337386,
      "eval_runtime": 22805.9951,
      "eval_samples_per_second": 0.009,
      "eval_steps_per_second": 0.002,
      "frac_reward_zero_std": 0.6938775510204082,
      "num_tokens": 2217192.0,
      "reward": 0.5408163265306123,
      "reward_std": 0.16411126146511157,
      "rewards/compute_reward/mean": 0.5408163265306123,
      "rewards/compute_reward/std": 0.16411126146511157,
      "step": 200
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04026442307692308,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.034542664885520935,
      "kl": 0.46225110441446304,
      "learning_rate": 1.5e-06,
      "loss": 0.0018,
      "num_tokens": 2226072.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 201
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04046474358974359,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.021351857110857964,
      "kl": 0.4879070967435837,
      "learning_rate": 1.4939418665842309e-06,
      "loss": 0.002,
      "num_tokens": 2234952.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 202
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.040665064102564104,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.904449939727783,
      "kl": 0.46766065806150436,
      "learning_rate": 1.4878596591387326e-06,
      "loss": 0.0019,
      "num_tokens": 2243832.0,
      "reward": 0.8125,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 203
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.040865384615384616,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.473119735717773,
      "kl": 0.46528438478708267,
      "learning_rate": 1.4817536741017151e-06,
      "loss": 0.0019,
      "num_tokens": 2252712.0,
      "reward": 0.8125,
      "reward_std": 0.4082317352294922,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 204
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04106570512820513,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.013064456172287464,
      "kl": 0.46919504553079605,
      "learning_rate": 1.4756242090702753e-06,
      "loss": 0.0019,
      "num_tokens": 2261592.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 205
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04126602564102564,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01172463595867157,
      "kl": 0.46472157537937164,
      "learning_rate": 1.4694715627858908e-06,
      "loss": 0.0019,
      "num_tokens": 2270472.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 206
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04146634615384615,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.549372673034668,
      "kl": 0.44779056310653687,
      "learning_rate": 1.4632960351198617e-06,
      "loss": 0.0018,
      "num_tokens": 2279352.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 207
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.041666666666666664,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.012319390662014484,
      "kl": 0.423983670771122,
      "learning_rate": 1.4570979270586943e-06,
      "loss": 0.0017,
      "num_tokens": 2288232.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 208
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.041866987179487176,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014685707166790962,
      "kl": 0.4125548005104065,
      "learning_rate": 1.4508775406894306e-06,
      "loss": 0.0017,
      "num_tokens": 2297112.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 209
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.042067307692307696,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.011721426621079445,
      "kl": 0.38092777878046036,
      "learning_rate": 1.4446351791849273e-06,
      "loss": 0.0015,
      "num_tokens": 2305992.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 210
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04226762820512821,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.011216741055250168,
      "kl": 0.3741166517138481,
      "learning_rate": 1.4383711467890773e-06,
      "loss": 0.0015,
      "num_tokens": 2314872.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 211
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04246794871794872,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.911791801452637,
      "kl": 0.3861362859606743,
      "learning_rate": 1.4320857488019824e-06,
      "loss": 0.0015,
      "num_tokens": 2323752.0,
      "reward": 0.5,
      "reward_std": 0.3535533845424652,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 212
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04266826923076923,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.871019124984741,
      "kl": 0.3671673685312271,
      "learning_rate": 1.4257792915650725e-06,
      "loss": 0.0015,
      "num_tokens": 2332632.0,
      "reward": 0.25,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 213
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.042868589743589744,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.8326685428619385,
      "kl": 0.3593239039182663,
      "learning_rate": 1.419452082446177e-06,
      "loss": 0.0014,
      "num_tokens": 2341512.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 214
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.043068910256410256,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.017604727298021317,
      "kl": 0.38986194878816605,
      "learning_rate": 1.4131044298245418e-06,
      "loss": 0.0016,
      "num_tokens": 2350392.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 215
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04326923076923077,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.469564437866211,
      "kl": 0.3725327253341675,
      "learning_rate": 1.4067366430758004e-06,
      "loss": 0.0015,
      "num_tokens": 2359272.0,
      "reward": 0.125,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 216
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04346955128205128,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.6093586087226868,
      "kl": 0.3573925197124481,
      "learning_rate": 1.400349032556895e-06,
      "loss": 0.0014,
      "num_tokens": 2368152.0,
      "reward": 0.6875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 217
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04366987179487179,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.40462327003479,
      "kl": 0.3738492876291275,
      "learning_rate": 1.393941909590951e-06,
      "loss": 0.0015,
      "num_tokens": 2377032.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 218
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.043870192307692304,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.48574450612068176,
      "kl": 0.3839896619319916,
      "learning_rate": 1.3875155864521028e-06,
      "loss": 0.0015,
      "num_tokens": 2385912.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 219
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.044070512820512824,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.6792774200439453,
      "kl": 0.40334972739219666,
      "learning_rate": 1.3810703763502743e-06,
      "loss": 0.0016,
      "num_tokens": 2394792.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 220
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.044270833333333336,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.009804892353713512,
      "kl": 0.3865264654159546,
      "learning_rate": 1.374606593415912e-06,
      "loss": 0.0015,
      "num_tokens": 2403672.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 221
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04447115384615385,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.008533602580428123,
      "kl": 0.38782164454460144,
      "learning_rate": 1.3681245526846781e-06,
      "loss": 0.0016,
      "num_tokens": 2412552.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 222
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04467147435897436,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.707383155822754,
      "kl": 0.4052330255508423,
      "learning_rate": 1.361624570082092e-06,
      "loss": 0.0016,
      "num_tokens": 2421432.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 223
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04487179487179487,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 6.930018901824951,
      "kl": 0.37737852334976196,
      "learning_rate": 1.3551069624081371e-06,
      "loss": 0.0015,
      "num_tokens": 2430312.0,
      "reward": 0.5625,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 224
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.045072115384615384,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.006927178241312504,
      "kl": 0.3862348794937134,
      "learning_rate": 1.3485720473218152e-06,
      "loss": 0.0015,
      "num_tokens": 2439192.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 225
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.045272435897435896,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 0.015935061499476433,
      "kl": 0.38616371154785156,
      "learning_rate": 1.3420201433256689e-06,
      "loss": 0.0015,
      "num_tokens": 2448072.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 226
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04547275641025641,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0508449077606201,
      "kl": 0.3866072744131088,
      "learning_rate": 1.3354515697502551e-06,
      "loss": 0.0015,
      "num_tokens": 2456952.0,
      "reward": 0.4375,
      "reward_std": 0.4082317352294922,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 227
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04567307692307692,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.6618845462799072,
      "kl": 0.3824535608291626,
      "learning_rate": 1.3288666467385831e-06,
      "loss": 0.0015,
      "num_tokens": 2465832.0,
      "reward": 0.25,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 228
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04587339743589743,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.6411046385765076,
      "kl": 0.5041236504912376,
      "learning_rate": 1.3222656952305111e-06,
      "loss": 0.002,
      "num_tokens": 2474712.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 229
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04607371794871795,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.018501494079828262,
      "kl": 0.4303128570318222,
      "learning_rate": 1.3156490369471024e-06,
      "loss": 0.0017,
      "num_tokens": 2483592.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 230
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.046274038461538464,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.061893939971924,
      "kl": 0.41076236218214035,
      "learning_rate": 1.3090169943749473e-06,
      "loss": 0.0016,
      "num_tokens": 2492472.0,
      "reward": 0.1875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 231
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.046474358974358976,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.540974140167236,
      "kl": 0.40235304832458496,
      "learning_rate": 1.3023698907504446e-06,
      "loss": 0.0016,
      "num_tokens": 2501352.0,
      "reward": 0.125,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 232
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04667467948717949,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.915604591369629,
      "kl": 0.4074578136205673,
      "learning_rate": 1.2957080500440467e-06,
      "loss": 0.0016,
      "num_tokens": 2510232.0,
      "reward": 0.75,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 233
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.046875,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.492879867553711,
      "kl": 0.4019653871655464,
      "learning_rate": 1.2890317969444716e-06,
      "loss": 0.0016,
      "num_tokens": 2519112.0,
      "reward": 0.75,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 234
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04707532051282051,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.010445642285048962,
      "kl": 0.40757326036691666,
      "learning_rate": 1.2823414568428766e-06,
      "loss": 0.0016,
      "num_tokens": 2527992.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 235
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.047275641025641024,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.015598655678331852,
      "kl": 0.40689441561698914,
      "learning_rate": 1.275637355816999e-06,
      "loss": 0.0016,
      "num_tokens": 2536872.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 236
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.047475961538461536,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014103200286626816,
      "kl": 0.41280630230903625,
      "learning_rate": 1.2689198206152656e-06,
      "loss": 0.0017,
      "num_tokens": 2545752.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 237
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04767628205128205,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.013156753033399582,
      "kl": 0.4309658110141754,
      "learning_rate": 1.2621891786408648e-06,
      "loss": 0.0017,
      "num_tokens": 2554632.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 238
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04787660256410257,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.0128845926374197,
      "kl": 0.42060084640979767,
      "learning_rate": 1.2554457579357905e-06,
      "loss": 0.0017,
      "num_tokens": 2563512.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 239
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04807692307692308,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.720545291900635,
      "kl": 0.4215758144855499,
      "learning_rate": 1.2486898871648551e-06,
      "loss": 0.0017,
      "num_tokens": 2572392.0,
      "reward": 0.25,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 240
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04827724358974359,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01492222212255001,
      "kl": 0.42492126673460007,
      "learning_rate": 1.2419218955996676e-06,
      "loss": 0.0017,
      "num_tokens": 2581272.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 241
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.048477564102564104,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.012079130858182907,
      "kl": 0.4157009422779083,
      "learning_rate": 1.23514211310259e-06,
      "loss": 0.0017,
      "num_tokens": 2590152.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 242
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.048677884615384616,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.5683152675628662,
      "kl": 0.4302680641412735,
      "learning_rate": 1.2283508701106558e-06,
      "loss": 0.0017,
      "num_tokens": 2599032.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 243
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04887820512820513,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.014624028466641903,
      "kl": 0.42716744542121887,
      "learning_rate": 1.2215484976194675e-06,
      "loss": 0.0017,
      "num_tokens": 2607912.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 244
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04907852564102564,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 7.9432291984558105,
      "kl": 0.44396263360977173,
      "learning_rate": 1.2147353271670632e-06,
      "loss": 0.0018,
      "num_tokens": 2616792.0,
      "reward": 0.4375,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 245
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.04927884615384615,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.099539279937744,
      "kl": 0.43849825859069824,
      "learning_rate": 1.207911690817759e-06,
      "loss": 0.0018,
      "num_tokens": 2625672.0,
      "reward": 0.1875,
      "reward_std": 0.4082317352294922,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 246
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.049479166666666664,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.018603794276714325,
      "kl": 0.44824808835983276,
      "learning_rate": 1.2010779211459648e-06,
      "loss": 0.0018,
      "num_tokens": 2634552.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 247
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.049679487179487176,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.01888052187860012,
      "kl": 0.4470982998609543,
      "learning_rate": 1.194234351219972e-06,
      "loss": 0.0018,
      "num_tokens": 2643432.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 248
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.049879807692307696,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.18263053894043,
      "kl": 0.4454714208841324,
      "learning_rate": 1.1873813145857248e-06,
      "loss": 0.0018,
      "num_tokens": 2652312.0,
      "reward": 0.875,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 249
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05008012820512821,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.06482016295194626,
      "kl": 0.4468122720718384,
      "learning_rate": 1.18051914525056e-06,
      "loss": 0.0018,
      "num_tokens": 2661192.0,
      "reward": 0.875,
      "reward_std": 0.3535533845424652,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 250
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05008012820512821,
      "eval_loss": -0.0013203199487179518,
      "eval_runtime": 25150.2942,
      "eval_samples_per_second": 0.008,
      "eval_steps_per_second": 0.001,
      "frac_reward_zero_std": 0.52,
      "num_tokens": 2883192.0,
      "reward": 0.4475,
      "reward_std": 0.2231314253807068,
      "rewards/compute_reward/mean": 0.4475,
      "rewards/compute_reward/std": 0.3909153664112091,
      "step": 250
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05028044871794872,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.6477034091949463,
      "kl": 0.4469309151172638,
      "learning_rate": 1.1736481776669305e-06,
      "loss": 0.1785,
      "num_tokens": 2892072.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 251
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05048076923076923,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.760794639587402,
      "kl": 0.4086172506213188,
      "learning_rate": 1.1667687467161023e-06,
      "loss": -0.1751,
      "num_tokens": 2900952.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 252
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.050681089743589744,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.538841247558594,
      "kl": 0.3545885533094406,
      "learning_rate": 1.1598811876918349e-06,
      "loss": 0.2621,
      "num_tokens": 2909832.0,
      "reward": 0.625,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 253
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.050881410256410256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 8.625150680541992,
      "kl": 0.3545182943344116,
      "learning_rate": 1.1529858362840382e-06,
      "loss": -0.0079,
      "num_tokens": 2918712.0,
      "reward": 0.5625,
      "reward_std": 0.49022960662841797,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 254
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05108173076923077,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.977151870727539,
      "kl": 0.35841332376003265,
      "learning_rate": 1.1460830285624116e-06,
      "loss": -0.1615,
      "num_tokens": 2927592.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 255
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05128205128205128,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.183256149291992,
      "kl": 0.3536309078335762,
      "learning_rate": 1.1391731009600653e-06,
      "loss": 0.0294,
      "num_tokens": 2936472.0,
      "reward": 0.1875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 256
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05148237179487179,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 4.800800323486328,
      "kl": 0.34444164484739304,
      "learning_rate": 1.1322563902571225e-06,
      "loss": -0.1518,
      "num_tokens": 2945352.0,
      "reward": 0.8125,
      "reward_std": 0.4082317352294922,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 257
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.051682692307692304,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 6.264692783355713,
      "kl": 0.3604939877986908,
      "learning_rate": 1.1253332335643042e-06,
      "loss": 0.8538,
      "num_tokens": 2954232.0,
      "reward": 0.375,
      "reward_std": 0.49871626496315,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 258
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.051883012820512824,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.3112211227417,
      "kl": 0.3770570084452629,
      "learning_rate": 1.1184039683065012e-06,
      "loss": -0.5489,
      "num_tokens": 2963112.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 259
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.052083333333333336,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 1.416114091873169,
      "kl": 0.36333227902650833,
      "learning_rate": 1.1114689322063254e-06,
      "loss": -0.0869,
      "num_tokens": 2971992.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 260
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05228365384615385,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.035257257521152496,
      "kl": 0.369166798889637,
      "learning_rate": 1.1045284632676535e-06,
      "loss": 0.0015,
      "num_tokens": 2980872.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 261
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05248397435897436,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.388102054595947,
      "kl": 0.3857538551092148,
      "learning_rate": 1.0975828997591495e-06,
      "loss": 0.2715,
      "num_tokens": 2989752.0,
      "reward": 0.875,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 262
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05268429487179487,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.785407066345215,
      "kl": 0.41566168516874313,
      "learning_rate": 1.0906325801977803e-06,
      "loss": -0.4033,
      "num_tokens": 2998632.0,
      "reward": 0.125,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 263
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.052884615384615384,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.76622200012207,
      "kl": 0.4089379236102104,
      "learning_rate": 1.0836778433323157e-06,
      "loss": 0.4066,
      "num_tokens": 3007512.0,
      "reward": 0.375,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 264
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.053084935897435896,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 7.712548732757568,
      "kl": 0.40098340809345245,
      "learning_rate": 1.0767190281268186e-06,
      "loss": -0.4418,
      "num_tokens": 3016392.0,
      "reward": 0.5625,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 265
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05328525641025641,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 13.751773834228516,
      "kl": 0.39320751279592514,
      "learning_rate": 1.069756473744125e-06,
      "loss": 0.8314,
      "num_tokens": 3025272.0,
      "reward": 0.4375,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 266
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05348557692307692,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.21595287322998,
      "kl": 0.3834692910313606,
      "learning_rate": 1.0627905195293135e-06,
      "loss": -0.9199,
      "num_tokens": 3034152.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 267
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05368589743589743,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 9.926980972290039,
      "kl": 0.39933696389198303,
      "learning_rate": 1.055821504993164e-06,
      "loss": 0.3133,
      "num_tokens": 3043032.0,
      "reward": 0.1875,
      "reward_std": 0.4082317352294922,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 268
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05388621794871795,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.699936866760254,
      "kl": 0.40862850844860077,
      "learning_rate": 1.0488497697956134e-06,
      "loss": 0.225,
      "num_tokens": 3051912.0,
      "reward": 0.375,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 269
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.054086538461538464,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.774573802947998,
      "kl": 0.4077768921852112,
      "learning_rate": 1.0418756537291995e-06,
      "loss": -0.5383,
      "num_tokens": 3060792.0,
      "reward": 0.375,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 270
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.054286858974358976,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 12.925228118896484,
      "kl": 0.39750829339027405,
      "learning_rate": 1.034899496702501e-06,
      "loss": -0.1938,
      "num_tokens": 3069672.0,
      "reward": 0.5625,
      "reward_std": 0.49022960662841797,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 271
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05448717948717949,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 12.457939147949219,
      "kl": 0.3762730360031128,
      "learning_rate": 1.0279216387235689e-06,
      "loss": 0.1326,
      "num_tokens": 3078552.0,
      "reward": 0.75,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 272
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0546875,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.428741455078125,
      "kl": 0.3705255761742592,
      "learning_rate": 1.020942419883357e-06,
      "loss": 0.2007,
      "num_tokens": 3087432.0,
      "reward": 0.875,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 273
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05488782051282051,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.15895938873291,
      "kl": 0.3654939755797386,
      "learning_rate": 1.0139621803391454e-06,
      "loss": 0.0157,
      "num_tokens": 3096312.0,
      "reward": 0.8125,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 274
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.055088141025641024,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 5.603805065155029,
      "kl": 0.35676612704992294,
      "learning_rate": 1.0069812602979615e-06,
      "loss": 0.1222,
      "num_tokens": 3105192.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 275
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.055288461538461536,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.030523981899023056,
      "kl": 0.3769182339310646,
      "learning_rate": 1e-06,
      "loss": 0.0015,
      "num_tokens": 3114072.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 276
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05548878205128205,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.522267818450928,
      "kl": 0.39615628123283386,
      "learning_rate": 9.930187397020384e-07,
      "loss": 0.1223,
      "num_tokens": 3122952.0,
      "reward": 0.1875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.1875,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 277
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05568910256410257,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 11.620308876037598,
      "kl": 0.3624560236930847,
      "learning_rate": 9.860378196608547e-07,
      "loss": 0.0618,
      "num_tokens": 3131832.0,
      "reward": 0.3125,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 278
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05588942307692308,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.2341694831848145,
      "kl": 0.4334682747721672,
      "learning_rate": 9.790575801166431e-07,
      "loss": -0.7096,
      "num_tokens": 3140712.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 279
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05608974358974359,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 7.147642612457275,
      "kl": 0.4480450674891472,
      "learning_rate": 9.720783612764313e-07,
      "loss": 0.532,
      "num_tokens": 3149592.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 280
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.056290064102564104,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.9908366203308105,
      "kl": 0.436921626329422,
      "learning_rate": 9.651005032974993e-07,
      "loss": -0.2397,
      "num_tokens": 3158472.0,
      "reward": 0.3125,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 281
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.056490384615384616,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 11.635929107666016,
      "kl": 0.4899734556674957,
      "learning_rate": 9.581243462708005e-07,
      "loss": -0.2242,
      "num_tokens": 3167352.0,
      "reward": 0.25,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 282
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05669070512820513,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.62132453918457,
      "kl": 0.42814522236585617,
      "learning_rate": 9.511502302043867e-07,
      "loss": 0.7031,
      "num_tokens": 3176232.0,
      "reward": 0.75,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 283
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05689102564102564,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.720637321472168,
      "kl": 0.45054399222135544,
      "learning_rate": 9.441784950068361e-07,
      "loss": -0.232,
      "num_tokens": 3185112.0,
      "reward": 0.875,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 284
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05709134615384615,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.327516555786133,
      "kl": 0.4447530210018158,
      "learning_rate": 9.372094804706866e-07,
      "loss": -0.5284,
      "num_tokens": 3193992.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 285
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.057291666666666664,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 13.709527015686035,
      "kl": 0.38602790236473083,
      "learning_rate": 9.302435262558747e-07,
      "loss": 1.0306,
      "num_tokens": 3202872.0,
      "reward": 0.6875,
      "reward_std": 0.44403791427612305,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 286
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.057491987179487176,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 3.679117202758789,
      "kl": 0.3744213953614235,
      "learning_rate": 9.232809718731813e-07,
      "loss": -0.4974,
      "num_tokens": 3211752.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 287
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.057692307692307696,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.686700820922852,
      "kl": 0.3607204779982567,
      "learning_rate": 9.163221566676847e-07,
      "loss": 0.3033,
      "num_tokens": 3220632.0,
      "reward": 0.6875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 288
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05789262820512821,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 4.125672340393066,
      "kl": 0.37398987263441086,
      "learning_rate": 9.093674198022199e-07,
      "loss": -0.3003,
      "num_tokens": 3229512.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 289
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05809294871794872,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.015799332410097122,
      "kl": 0.36872173845767975,
      "learning_rate": 9.024171002408506e-07,
      "loss": 0.0015,
      "num_tokens": 3238392.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 290
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05829326923076923,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 9.486305236816406,
      "kl": 0.3870535045862198,
      "learning_rate": 8.954715367323466e-07,
      "loss": -0.3607,
      "num_tokens": 3247272.0,
      "reward": 0.4375,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 291
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.058493589743589744,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 6.253905296325684,
      "kl": 0.3818277567625046,
      "learning_rate": 8.885310677936746e-07,
      "loss": 0.2583,
      "num_tokens": 3256152.0,
      "reward": 0.5625,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 292
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.058693910256410256,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.854904651641846,
      "kl": 0.3843723237514496,
      "learning_rate": 8.81596031693499e-07,
      "loss": 0.0186,
      "num_tokens": 3265032.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 293
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05889423076923077,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.7960243225097656,
      "kl": 0.37519288808107376,
      "learning_rate": 8.746667664356955e-07,
      "loss": 0.0899,
      "num_tokens": 3273912.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 294
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05909455128205128,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.167747974395752,
      "kl": 0.3417524918913841,
      "learning_rate": 8.677436097428774e-07,
      "loss": -0.6496,
      "num_tokens": 3282792.0,
      "reward": 0.75,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 295
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.05929487179487179,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 9.317471504211426,
      "kl": 0.35206327587366104,
      "learning_rate": 8.608268990399348e-07,
      "loss": 0.4844,
      "num_tokens": 3291672.0,
      "reward": 0.75,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 296
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.059495192307692304,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 6.916869640350342,
      "kl": 0.3519115597009659,
      "learning_rate": 8.539169714375885e-07,
      "loss": -0.3136,
      "num_tokens": 3300552.0,
      "reward": 0.5,
      "reward_std": 0.5175491571426392,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 297
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.059695512820512824,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 5.388540267944336,
      "kl": 0.35239919275045395,
      "learning_rate": 8.47014163715962e-07,
      "loss": 1.4196,
      "num_tokens": 3309432.0,
      "reward": 0.5,
      "reward_std": 0.5345224738121033,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 298
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.059895833333333336,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 9.753130912780762,
      "kl": 0.3539476618170738,
      "learning_rate": 8.401188123081652e-07,
      "loss": -0.8778,
      "num_tokens": 3318312.0,
      "reward": 0.75,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 299
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06009615384615385,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 6.6426801681518555,
      "kl": 0.35463206470012665,
      "learning_rate": 8.332312532838978e-07,
      "loss": 0.0942,
      "num_tokens": 3327192.0,
      "reward": 0.75,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 300
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06009615384615385,
      "eval_loss": -0.005364453885704279,
      "eval_runtime": 36668.8637,
      "eval_samples_per_second": 0.005,
      "eval_steps_per_second": 0.0,
      "frac_reward_zero_std": 0.52,
      "num_tokens": 3549192.0,
      "reward": 0.3475,
      "reward_std": 0.2188346815109253,
      "rewards/compute_reward/mean": 0.3475,
      "rewards/compute_reward/std": 0.35017398118972776,
      "step": 300
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06029647435897436,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 4.439915657043457,
      "kl": 0.34843526035547256,
      "learning_rate": 8.263518223330696e-07,
      "loss": 0.424,
      "num_tokens": 3558072.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 301
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06049679487179487,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.283510684967041,
      "kl": 0.356955386698246,
      "learning_rate": 8.194808547494401e-07,
      "loss": -0.1336,
      "num_tokens": 3566952.0,
      "reward": 0.375,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 302
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.060697115384615384,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.668262004852295,
      "kl": 0.3441893681883812,
      "learning_rate": 8.126186854142751e-07,
      "loss": 0.4898,
      "num_tokens": 3575832.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 303
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.060897435897435896,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 10.111356735229492,
      "kl": 0.38992520421743393,
      "learning_rate": 8.057656487800282e-07,
      "loss": -0.3519,
      "num_tokens": 3584712.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 304
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06109775641025641,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 10.168166160583496,
      "kl": 0.3918253928422928,
      "learning_rate": 7.989220788540355e-07,
      "loss": -0.1719,
      "num_tokens": 3593592.0,
      "reward": 0.4375,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 305
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06129807692307692,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 14.932116508483887,
      "kl": 0.42335211485624313,
      "learning_rate": 7.920883091822408e-07,
      "loss": 0.2871,
      "num_tokens": 3602472.0,
      "reward": 0.625,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 306
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06149839743589743,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 7.603961944580078,
      "kl": 0.41014522314071655,
      "learning_rate": 7.852646728329367e-07,
      "loss": -0.1103,
      "num_tokens": 3611352.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 307
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06169871794871795,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04095939174294472,
      "kl": 0.41697999089956284,
      "learning_rate": 7.784515023805327e-07,
      "loss": 0.0017,
      "num_tokens": 3620232.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 308
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.061899038461538464,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.4544095993042,
      "kl": 0.4006544202566147,
      "learning_rate": 7.716491298893441e-07,
      "loss": -0.2684,
      "num_tokens": 3629112.0,
      "reward": 0.625,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 309
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.062099358974358976,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.377477645874023,
      "kl": 0.43658439815044403,
      "learning_rate": 7.648578868974099e-07,
      "loss": -0.1959,
      "num_tokens": 3637992.0,
      "reward": 0.75,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 310
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06229967948717949,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.106303215026855,
      "kl": 0.43939898163080215,
      "learning_rate": 7.580781044003324e-07,
      "loss": 0.2356,
      "num_tokens": 3646872.0,
      "reward": 0.25,
      "reward_std": 0.26726123690605164,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 311
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0625,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 11.934514999389648,
      "kl": 0.48612669110298157,
      "learning_rate": 7.513101128351453e-07,
      "loss": -0.3042,
      "num_tokens": 3655752.0,
      "reward": 0.125,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.125,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 312
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06270032051282051,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.370943069458008,
      "kl": 0.4450509250164032,
      "learning_rate": 7.445542420642096e-07,
      "loss": 0.7228,
      "num_tokens": 3664632.0,
      "reward": 0.6875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 313
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06290064102564102,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 12.907133102416992,
      "kl": 0.4375302121043205,
      "learning_rate": 7.378108213591354e-07,
      "loss": -0.1794,
      "num_tokens": 3673512.0,
      "reward": 0.875,
      "reward_std": 0.3535533845424652,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 314
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06310096153846154,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 19.36090660095215,
      "kl": 0.4966515228152275,
      "learning_rate": 7.310801793847343e-07,
      "loss": 0.2802,
      "num_tokens": 3682392.0,
      "reward": 0.25,
      "reward_std": 0.4355512857437134,
      "rewards/compute_reward/mean": 0.25,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 315
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06330128205128205,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 13.052001953125,
      "kl": 0.5096459239721298,
      "learning_rate": 7.243626441830009e-07,
      "loss": -0.0347,
      "num_tokens": 3691272.0,
      "reward": 0.3125,
      "reward_std": 0.49022960662841797,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 316
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06350160256410256,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 16.242658615112305,
      "kl": 0.5109384208917618,
      "learning_rate": 7.176585431571233e-07,
      "loss": -0.5929,
      "num_tokens": 3700152.0,
      "reward": 0.5625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 317
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06370192307692307,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.50827693939209,
      "kl": 0.5058481469750404,
      "learning_rate": 7.109682030555282e-07,
      "loss": 0.7604,
      "num_tokens": 3709032.0,
      "reward": 0.625,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 318
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06390224358974358,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.333704948425293,
      "kl": 0.47000084072351456,
      "learning_rate": 7.042919499559536e-07,
      "loss": -0.0496,
      "num_tokens": 3717912.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 319
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0641025641025641,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 11.68780517578125,
      "kl": 0.4798329174518585,
      "learning_rate": 6.976301092495556e-07,
      "loss": 0.5008,
      "num_tokens": 3726792.0,
      "reward": 0.3125,
      "reward_std": 0.44403791427612305,
      "rewards/compute_reward/mean": 0.3125,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 320
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06430288461538461,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 18.300569534301758,
      "kl": 0.4934264048933983,
      "learning_rate": 6.909830056250526e-07,
      "loss": -1.5204,
      "num_tokens": 3735672.0,
      "reward": 0.4375,
      "reward_std": 0.4082317352294922,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 321
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06450320512820513,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 16.41554069519043,
      "kl": 0.49192648380994797,
      "learning_rate": 6.843509630528976e-07,
      "loss": 1.4819,
      "num_tokens": 3744552.0,
      "reward": 0.5,
      "reward_std": 0.4629100561141968,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 322
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06470352564102565,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 10.066646575927734,
      "kl": 0.49376900494098663,
      "learning_rate": 6.77734304769489e-07,
      "loss": -0.4545,
      "num_tokens": 3753432.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 323
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06490384615384616,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 10.097261428833008,
      "kl": 0.5123012214899063,
      "learning_rate": 6.711333532614167e-07,
      "loss": -0.3514,
      "num_tokens": 3762312.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 324
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06510416666666667,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 10.803947448730469,
      "kl": 0.5209080129861832,
      "learning_rate": 6.645484302497451e-07,
      "loss": -0.5555,
      "num_tokens": 3771192.0,
      "reward": 0.4375,
      "reward_std": 0.49022960662841797,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 325
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06530448717948718,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 11.348974227905273,
      "kl": 0.49841564148664474,
      "learning_rate": 6.579798566743313e-07,
      "loss": -0.0968,
      "num_tokens": 3780072.0,
      "reward": 0.5625,
      "reward_std": 0.5260357856750488,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 326
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0655048076923077,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 17.2227840423584,
      "kl": 0.5217346847057343,
      "learning_rate": 6.51427952678185e-07,
      "loss": 1.1984,
      "num_tokens": 3788952.0,
      "reward": 0.375,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.375,
      "rewards/compute_reward/std": 0.5,
      "step": 327
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06570512820512821,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 10.868518829345703,
      "kl": 0.5167401880025864,
      "learning_rate": 6.448930375918631e-07,
      "loss": -0.0703,
      "num_tokens": 3797832.0,
      "reward": 0.5,
      "reward_std": 0.5345224738121033,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 328
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06590544871794872,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 18.512907028198242,
      "kl": 0.5056992769241333,
      "learning_rate": 6.383754299179078e-07,
      "loss": -0.4656,
      "num_tokens": 3806712.0,
      "reward": 0.75,
      "reward_std": 0.4629100561141968,
      "rewards/compute_reward/mean": 0.75,
      "rewards/compute_reward/std": 0.44721361994743347,
      "step": 329
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06610576923076923,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 13.673883438110352,
      "kl": 0.516699343919754,
      "learning_rate": 6.31875447315322e-07,
      "loss": 0.0021,
      "num_tokens": 3815592.0,
      "reward": 0.875,
      "reward_std": 0.3535533845424652,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 330
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06630608974358974,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 14.64022445678711,
      "kl": 0.4903121292591095,
      "learning_rate": 6.253934065840879e-07,
      "loss": 0.5009,
      "num_tokens": 3824472.0,
      "reward": 0.6875,
      "reward_std": 0.44403791427612305,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 331
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06650641025641026,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 12.848752975463867,
      "kl": 0.48798027634620667,
      "learning_rate": 6.189296236497259e-07,
      "loss": -0.5573,
      "num_tokens": 3833352.0,
      "reward": 0.8125,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.8125,
      "rewards/compute_reward/std": 0.40311288833618164,
      "step": 332
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06670673076923077,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.995890140533447,
      "kl": 0.496772900223732,
      "learning_rate": 6.124844135478971e-07,
      "loss": -0.2911,
      "num_tokens": 3842232.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 333
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06690705128205128,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 9.475105285644531,
      "kl": 0.4880137965083122,
      "learning_rate": 6.060580904090489e-07,
      "loss": 0.002,
      "num_tokens": 3851112.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 334
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06710737179487179,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 7.369569778442383,
      "kl": 0.4540814310312271,
      "learning_rate": 5.996509674431051e-07,
      "loss": 0.7087,
      "num_tokens": 3859992.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 335
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0673076923076923,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 6.226390838623047,
      "kl": 0.4414127245545387,
      "learning_rate": 5.932633569241999e-07,
      "loss": 0.0901,
      "num_tokens": 3868872.0,
      "reward": 0.0625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.0625,
      "rewards/compute_reward/std": 0.25,
      "step": 336
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06750801282051282,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 4.00337028503418,
      "kl": 0.4680175557732582,
      "learning_rate": 5.868955701754583e-07,
      "loss": -0.4399,
      "num_tokens": 3877752.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 337
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06770833333333333,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.04524607211351395,
      "kl": 0.4537159204483032,
      "learning_rate": 5.805479175538228e-07,
      "loss": 0.0018,
      "num_tokens": 3886632.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 338
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06790865384615384,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.417373538017273,
      "kl": 0.45146308839321136,
      "learning_rate": 5.742207084349273e-07,
      "loss": -0.2633,
      "num_tokens": 3895512.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 339
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06810897435897435,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 5.8812947273254395,
      "kl": 0.46996302157640457,
      "learning_rate": 5.679142511980175e-07,
      "loss": -0.1748,
      "num_tokens": 3904392.0,
      "reward": 0.5625,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.5625,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 340
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06830929487179487,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 1.6094920635223389,
      "kl": 0.4542356953024864,
      "learning_rate": 5.616288532109224e-07,
      "loss": 0.0902,
      "num_tokens": 3913272.0,
      "reward": 0.4375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.4375,
      "rewards/compute_reward/std": 0.5123475790023804,
      "step": 341
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06850961538461539,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 2.4263854026794434,
      "kl": 0.44890759885311127,
      "learning_rate": 5.553648208150728e-07,
      "loss": 0.3552,
      "num_tokens": 3922152.0,
      "reward": 0.5,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.5,
      "rewards/compute_reward/std": 0.5163977742195129,
      "step": 342
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.0687099358974359,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.750292778015137,
      "kl": 0.4503125175833702,
      "learning_rate": 5.491224593105694e-07,
      "loss": -0.6731,
      "num_tokens": 3931032.0,
      "reward": 0.625,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 343
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06891025641025642,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 4.980900287628174,
      "kl": 0.4370996654033661,
      "learning_rate": 5.42902072941306e-07,
      "loss": 1.2166,
      "num_tokens": 3939912.0,
      "reward": 0.625,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.625,
      "rewards/compute_reward/std": 0.5,
      "step": 344
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06911057692307693,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 8.993419647216797,
      "kl": 0.4312536120414734,
      "learning_rate": 5.367039648801385e-07,
      "loss": -0.1848,
      "num_tokens": 3948792.0,
      "reward": 0.9375,
      "reward_std": 0.1767766922712326,
      "rewards/compute_reward/mean": 0.9375,
      "rewards/compute_reward/std": 0.25,
      "step": 345
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06931089743589744,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 1.5246102809906006,
      "kl": 0.39856231212615967,
      "learning_rate": 5.305284372141095e-07,
      "loss": -0.3519,
      "num_tokens": 3957672.0,
      "reward": 1.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 1.0,
      "rewards/compute_reward/std": 0.0,
      "step": 346
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06951121794871795,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 2.0591368675231934,
      "kl": 0.40252938121557236,
      "learning_rate": 5.243757909297246e-07,
      "loss": 0.062,
      "num_tokens": 3966552.0,
      "reward": 0.6875,
      "reward_std": 0.2587745785713196,
      "rewards/compute_reward/mean": 0.6875,
      "rewards/compute_reward/std": 0.4787135720252991,
      "step": 347
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06971153846153846,
      "frac_reward_zero_std": 0.5,
      "grad_norm": 3.7395071983337402,
      "kl": 0.3975580483675003,
      "learning_rate": 5.182463258982846e-07,
      "loss": 0.4812,
      "num_tokens": 3975432.0,
      "reward": 0.875,
      "reward_std": 0.2314550280570984,
      "rewards/compute_reward/mean": 0.875,
      "rewards/compute_reward/std": 0.3415650427341461,
      "step": 348
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.06991185897435898,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 2.202305555343628,
      "kl": 0.39678341895341873,
      "learning_rate": 5.121403408612671e-07,
      "loss": -0.5384,
      "num_tokens": 3984312.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 349
    },
    {
      "completion_length": 512.0,
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.07011217948717949,
      "frac_reward_zero_std": 1.0,
      "grad_norm": 0.02550048567354679,
      "kl": 0.3984368219971657,
      "learning_rate": 5.060581334157692e-07,
      "loss": 0.0016,
      "num_tokens": 3993192.0,
      "reward": 0.0,
      "reward_std": 0.0,
      "rewards/compute_reward/mean": 0.0,
      "rewards/compute_reward/std": 0.0,
      "step": 350
    },
    {
      "completions/clipped_ratio": 1.0,
      "completions/max_length": 512.0,
      "completions/max_terminated_length": 0.0,
      "completions/mean_length": 512.0,
      "completions/mean_terminated_length": 0.0,
      "completions/min_length": 512.0,
      "completions/min_terminated_length": 0.0,
      "epoch": 0.07011217948717949,
      "eval_loss": 0.0017489355523139238,
      "eval_runtime": 22558.8436,
      "eval_samples_per_second": 0.009,
      "eval_steps_per_second": 0.001,
      "frac_reward_zero_std": 0.5,
      "num_tokens": 4215192.0,
      "reward": 0.3075,
      "reward_std": 0.2354060286283493,
      "rewards/compute_reward/mean": 0.3075,
      "rewards/compute_reward/std": 0.3394659686088562,
      "step": 350
    }
  ],
  "logging_steps": 1,
  "max_steps": 500,
  "num_input_tokens_seen": 4215192,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": false
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 4,
  "trial_name": null,
  "trial_params": null
}