{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.011428571428571429,
  "eval_steps": 500,
  "global_step": 20,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio": 0.0,
      "completion_length": 826.625,
      "epoch": 0.0005714285714285715,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": 0.003743603825569153,
      "reward": -0.2634471617639065,
      "reward_std": 0.18880490399897099,
      "rewards/cosine_scaled_reward": -0.13172357995063066,
      "rewards/format_reward": 0.0,
      "step": 1
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 856.25,
      "epoch": 0.001142857142857143,
      "kl": 0.0,
      "learning_rate": 1e-06,
      "loss": 5.21540641784668e-08,
      "reward": -0.37787145748734474,
      "reward_std": 0.3398940674960613,
      "rewards/cosine_scaled_reward": -0.18893574364483356,
      "rewards/format_reward": 0.0,
      "step": 2
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 928.875,
      "epoch": 0.0017142857142857142,
      "kl": 0.002269744873046875,
      "learning_rate": 2e-06,
      "loss": 9.12398099899292e-05,
      "reward": -0.4205120652914047,
      "reward_std": 0.3639860153198242,
      "rewards/cosine_scaled_reward": -0.21025604009628296,
      "rewards/format_reward": 0.0,
      "step": 3
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 916.0,
      "epoch": 0.002285714285714286,
      "kl": 0.005883216857910156,
      "learning_rate": 1.986326977710987e-06,
      "loss": 0.00023540854454040527,
      "reward": -0.3235983597114682,
      "reward_std": 0.27216803655028343,
      "rewards/cosine_scaled_reward": -0.1617991798557341,
      "rewards/format_reward": 0.0,
      "step": 4
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 924.6875,
      "epoch": 0.002857142857142857,
      "kl": 0.0016026496887207031,
      "learning_rate": 1.9457233587073175e-06,
      "loss": -0.0842575952410698,
      "reward": -0.32677557691931725,
      "reward_std": 0.2703730836510658,
      "rewards/cosine_scaled_reward": -0.16338778473436832,
      "rewards/format_reward": 0.0,
      "step": 5
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 899.375,
      "epoch": 0.0034285714285714284,
      "kl": 0.010833740234375,
      "learning_rate": 1.8794228634059947e-06,
      "loss": -0.0006134212017059326,
      "reward": -0.4203099086880684,
      "reward_std": 0.3194398656487465,
      "rewards/cosine_scaled_reward": -0.21015496738255024,
      "rewards/format_reward": 0.0,
      "step": 6
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 973.5,
      "epoch": 0.004,
      "kl": 0.029541015625,
      "learning_rate": 1.78943999880708e-06,
      "loss": 0.0011770054697990417,
      "reward": -0.324583163484931,
      "reward_std": 0.22306307777762413,
      "rewards/cosine_scaled_reward": -0.1622915817424655,
      "rewards/format_reward": 0.0,
      "step": 7
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1024.0,
      "epoch": 0.004571428571428572,
      "kl": 0.047206878662109375,
      "learning_rate": 1.6785088487178854e-06,
      "loss": 0.0018889307975769043,
      "reward": -0.22153206076472998,
      "reward_std": 0.27774741500616074,
      "rewards/cosine_scaled_reward": -0.11076603084802628,
      "rewards/format_reward": 0.0,
      "step": 8
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1012.625,
      "epoch": 0.005142857142857143,
      "kl": 0.004932403564453125,
      "learning_rate": 1.55e-06,
      "loss": 0.0001969374716281891,
      "reward": -0.2062576599419117,
      "reward_std": 0.2492200918495655,
      "rewards/cosine_scaled_reward": -0.10312882997095585,
      "rewards/format_reward": 0.0,
      "step": 9
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1024.0,
      "epoch": 0.005714285714285714,
      "kl": 0.0265045166015625,
      "learning_rate": 1.4078181289931019e-06,
      "loss": 0.0010615885257720947,
      "reward": -0.3071232885122299,
      "reward_std": 0.4035240039229393,
      "rewards/cosine_scaled_reward": -0.15356164425611496,
      "rewards/format_reward": 0.0,
      "step": 10
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1024.0,
      "epoch": 0.006285714285714286,
      "kl": 0.016486644744873047,
      "learning_rate": 1.2562833599002374e-06,
      "loss": 0.0006595328450202942,
      "reward": -0.2060939408838749,
      "reward_std": 0.17943128757178783,
      "rewards/cosine_scaled_reward": -0.1030469723045826,
      "rewards/format_reward": 0.0,
      "step": 11
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 998.625,
      "epoch": 0.006857142857142857,
      "kl": 0.03231048583984375,
      "learning_rate": 1.1e-06,
      "loss": 0.00129680335521698,
      "reward": -0.2936643697321415,
      "reward_std": 0.3133760243654251,
      "rewards/cosine_scaled_reward": -0.14683218486607075,
      "rewards/format_reward": 0.0,
      "step": 12
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 854.875,
      "epoch": 0.0074285714285714285,
      "kl": 0.011768341064453125,
      "learning_rate": 9.437166400997627e-07,
      "loss": 0.0004699230194091797,
      "reward": -0.37432078272104263,
      "reward_std": 0.3915751725435257,
      "rewards/cosine_scaled_reward": -0.18716039136052132,
      "rewards/format_reward": 0.0,
      "step": 13
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1012.0,
      "epoch": 0.008,
      "kl": 0.07757759094238281,
      "learning_rate": 7.921818710068981e-07,
      "loss": 0.003101266920566559,
      "reward": -0.22097227722406387,
      "reward_std": 0.21915528364479542,
      "rewards/cosine_scaled_reward": -0.11048614233732224,
      "rewards/format_reward": 0.0,
      "step": 14
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 976.375,
      "epoch": 0.008571428571428572,
      "kl": 0.027825355529785156,
      "learning_rate": 6.500000000000002e-07,
      "loss": 0.0011118575930595398,
      "reward": -0.2121176845394075,
      "reward_std": 0.2806076686247252,
      "rewards/cosine_scaled_reward": -0.10605884226970375,
      "rewards/format_reward": 0.0,
      "step": 15
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 915.375,
      "epoch": 0.009142857142857144,
      "kl": 0.02098846435546875,
      "learning_rate": 5.214911512821145e-07,
      "loss": -0.011211458593606949,
      "reward": -0.5936953723430634,
      "reward_std": 0.4250611439347267,
      "rewards/cosine_scaled_reward": -0.2968476861715317,
      "rewards/format_reward": 0.0,
      "step": 16
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 971.5,
      "epoch": 0.009714285714285713,
      "kl": 0.035379886627197266,
      "learning_rate": 4.105600011929199e-07,
      "loss": -0.00897778570652008,
      "reward": -0.19128201343119144,
      "reward_std": 0.1637940350919962,
      "rewards/cosine_scaled_reward": -0.09564100485295057,
      "rewards/format_reward": 0.0,
      "step": 17
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 835.375,
      "epoch": 0.010285714285714285,
      "kl": 0.0531005859375,
      "learning_rate": 3.2057713659400516e-07,
      "loss": -0.03545624762773514,
      "reward": -0.35267514176666737,
      "reward_std": 0.257406591437757,
      "rewards/cosine_scaled_reward": -0.17633757088333368,
      "rewards/format_reward": 0.0,
      "step": 18
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1016.125,
      "epoch": 0.010857142857142857,
      "kl": 0.03937530517578125,
      "learning_rate": 2.542766412926825e-07,
      "loss": 0.0015729628503322601,
      "reward": -0.19047284871339798,
      "reward_std": 0.18389804009348154,
      "rewards/cosine_scaled_reward": -0.09523642808198929,
      "rewards/format_reward": 0.0,
      "step": 19
    },
    {
      "clip_ratio": 0.0,
      "completion_length": 1024.0,
      "epoch": 0.011428571428571429,
      "kl": 0.0442662239074707,
      "learning_rate": 2.136730222890128e-07,
      "loss": 0.0017768293619155884,
      "reward": -0.23700407380238175,
      "reward_std": 0.27234716434031725,
      "rewards/cosine_scaled_reward": -0.11850203666836023,
      "rewards/format_reward": 0.0,
      "step": 20
    },
    {
      "epoch": 0.011428571428571429,
      "step": 20,
      "total_flos": 0.0,
      "train_loss": -0.00610662829130888,
      "train_runtime": 18644.4882,
      "train_samples_per_second": 0.017,
      "train_steps_per_second": 0.001
    }
  ],
  "logging_steps": 1,
  "max_steps": 20,
  "num_input_tokens_seen": 0,
  "num_train_epochs": 1,
  "save_steps": 50,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 2,
  "trial_name": null,
  "trial_params": null
}