{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.0005,
  "eval_steps": 500,
  "global_step": 50,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8966.0,
      "completions/max_terminated_length": 8966.0,
      "completions/mean_length": 7392.1875,
      "completions/mean_terminated_length": 7392.1875,
      "completions/min_length": 892.0,
      "completions/min_terminated_length": 892.0,
      "entropy": 0.11410779180005193,
      "epoch": 1e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.795484185218811,
      "kl": 0.0,
      "learning_rate": 0.0,
      "loss": -0.2193,
      "num_tokens": 263588.0,
      "reward": -0.3529798984527588,
      "reward_std": 0.3798896074295044,
      "rewards/rollout_reward_func/mean": -0.3529798984527588,
      "rewards/rollout_reward_func/std": 0.4275760054588318,
      "sampling/importance_sampling_ratio/max": 1.9381288290023804,
      "sampling/importance_sampling_ratio/mean": 0.9994939565658569,
      "sampling/importance_sampling_ratio/min": 0.48449021577835083,
      "sampling/sampling_logp_difference/max": 0.7246580123901367,
      "sampling/sampling_logp_difference/mean": 0.013468129560351372,
      "step": 1,
      "step_time": 93.399304126
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "entropy": 0.11410779180005193,
      "epoch": 2e-05,
      "grad_norm": 1.7967665195465088,
      "kl": 0.0,
      "learning_rate": 2.8571428571428573e-06,
      "loss": -0.2193,
      "step": 2,
      "step_time": 27.58208410200001
    },
    {
      "clip_ratio/high_max": 0.002206696954090148,
      "clip_ratio/high_mean": 0.001103348477045074,
      "clip_ratio/low_mean": 0.0021773561311420053,
      "clip_ratio/low_min": 0.0007183908019214869,
      "clip_ratio/region_mean": 0.0032807046081870794,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8789.0,
      "completions/max_terminated_length": 8789.0,
      "completions/mean_length": 7273.3125,
      "completions/mean_terminated_length": 7273.3125,
      "completions/min_length": 1689.0,
      "completions/min_terminated_length": 1689.0,
      "entropy": 0.08583545382134616,
      "epoch": 3e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1726843118667603,
      "kl": 0.0017027981803039438,
      "learning_rate": 5.7142857142857145e-06,
      "loss": -0.0032,
      "num_tokens": 523711.0,
      "reward": -0.3553134799003601,
      "reward_std": 0.48743048310279846,
      "rewards/rollout_reward_func/mean": -0.3553134799003601,
      "rewards/rollout_reward_func/std": 0.5032535195350647,
      "sampling/importance_sampling_ratio/max": 2.636791944503784,
      "sampling/importance_sampling_ratio/mean": 1.0005749464035034,
      "sampling/importance_sampling_ratio/min": 0.6534329652786255,
      "sampling/sampling_logp_difference/max": 0.9695630073547363,
      "sampling/sampling_logp_difference/mean": 0.008272429928183556,
      "step": 3,
      "step_time": 91.38279614700019
    },
    {
      "clip_ratio/high_max": 0.00520685565425083,
      "clip_ratio/high_mean": 0.002603427827125415,
      "clip_ratio/low_mean": 0.0017682212928775698,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004371649120002985,
      "entropy": 0.08481810381636024,
      "epoch": 4e-05,
      "grad_norm": 2.2690632343292236,
      "kl": 0.0009417296496394556,
      "learning_rate": 8.571428571428573e-06,
      "loss": -0.0051,
      "step": 4,
      "step_time": 26.98271870500014
    },
    {
      "clip_ratio/high_max": 0.004254478495568037,
      "clip_ratio/high_mean": 0.0021272392477840185,
      "clip_ratio/low_mean": 0.002160204341635108,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0042874435894191265,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8986.0,
      "completions/max_terminated_length": 8986.0,
      "completions/mean_length": 7767.59375,
      "completions/mean_terminated_length": 7767.59375,
      "completions/min_length": 300.0,
      "completions/min_terminated_length": 300.0,
      "entropy": 0.10082893958315253,
      "epoch": 5e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3848886489868164,
      "kl": 0.0014980694850237342,
      "learning_rate": 1.1428571428571429e-05,
      "loss": -0.1089,
      "num_tokens": 799123.0,
      "reward": -0.4331575036048889,
      "reward_std": 0.3397783935070038,
      "rewards/rollout_reward_func/mean": -0.4331575036048889,
      "rewards/rollout_reward_func/std": 0.37834790349006653,
      "sampling/importance_sampling_ratio/max": 1.6952898502349854,
      "sampling/importance_sampling_ratio/mean": 0.9987826347351074,
      "sampling/importance_sampling_ratio/min": 0.2250514030456543,
      "sampling/sampling_logp_difference/max": 1.4914264678955078,
      "sampling/sampling_logp_difference/mean": 0.010691944509744644,
      "step": 5,
      "step_time": 94.93945038100003
    },
    {
      "clip_ratio/high_max": 0.006353594711981714,
      "clip_ratio/high_mean": 0.003176797355990857,
      "clip_ratio/low_mean": 0.0018021696887444705,
      "clip_ratio/low_min": 0.0007183908019214869,
      "clip_ratio/region_mean": 0.004978967044735327,
      "entropy": 0.10040339734405279,
      "epoch": 6e-05,
      "grad_norm": 1.3835108280181885,
      "kl": 0.00129023771660286,
      "learning_rate": 1.4285714285714285e-05,
      "loss": -0.1114,
      "step": 6,
      "step_time": 27.51428913800021
    },
    {
      "clip_ratio/high_max": 0.007119529938790947,
      "clip_ratio/high_mean": 0.0035597649693954736,
      "clip_ratio/low_mean": 0.0018344622512813658,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0053942272788845,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9295.0,
      "completions/max_terminated_length": 9295.0,
      "completions/mean_length": 8119.90625,
      "completions/mean_terminated_length": 8119.90625,
      "completions/min_length": 425.0,
      "completions/min_terminated_length": 425.0,
      "entropy": 0.08391103229951113,
      "epoch": 7e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.6203776597976685,
      "kl": 0.0014097213361310423,
      "learning_rate": 1.7142857142857145e-05,
      "loss": -0.1002,
      "num_tokens": 1086692.0,
      "reward": -0.5209137201309204,
      "reward_std": 0.2494075447320938,
      "rewards/rollout_reward_func/mean": -0.5209137201309204,
      "rewards/rollout_reward_func/std": 0.2681976854801178,
      "sampling/importance_sampling_ratio/max": 1.7586421966552734,
      "sampling/importance_sampling_ratio/mean": 1.0001193284988403,
      "sampling/importance_sampling_ratio/min": 0.5591319799423218,
      "sampling/sampling_logp_difference/max": 0.5813696980476379,
      "sampling/sampling_logp_difference/mean": 0.010238477028906345,
      "step": 7,
      "step_time": 99.45176190400002
    },
    {
      "clip_ratio/high_max": 0.0050131616299040616,
      "clip_ratio/high_mean": 0.0025065808149520308,
      "clip_ratio/low_mean": 0.0048430965398438275,
      "clip_ratio/low_min": 0.0014367816038429737,
      "clip_ratio/region_mean": 0.00734967747121118,
      "entropy": 0.08137266687117517,
      "epoch": 8e-05,
      "grad_norm": 1.417209267616272,
      "kl": 0.0033996296788245672,
      "learning_rate": 2e-05,
      "loss": -0.1028,
      "step": 8,
      "step_time": 29.014337608000233
    },
    {
      "clip_ratio/high_max": 0.0042058578110300004,
      "clip_ratio/high_mean": 0.0024501511361449957,
      "clip_ratio/low_mean": 0.0024779988452792168,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0049281499814242125,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9264.0,
      "completions/max_terminated_length": 9264.0,
      "completions/mean_length": 7546.03125,
      "completions/mean_terminated_length": 7546.03125,
      "completions/min_length": 860.0,
      "completions/min_terminated_length": 860.0,
      "entropy": 0.08567450987175107,
      "epoch": 9e-05,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.5940141677856445,
      "kl": 0.010301298352715094,
      "learning_rate": 2.2857142857142858e-05,
      "loss": -0.3628,
      "num_tokens": 1355432.0,
      "reward": -0.40637093782424927,
      "reward_std": 0.3687950074672699,
      "rewards/rollout_reward_func/mean": -0.40637093782424927,
      "rewards/rollout_reward_func/std": 0.3848126232624054,
      "sampling/importance_sampling_ratio/max": 1.819852352142334,
      "sampling/importance_sampling_ratio/mean": 1.0000147819519043,
      "sampling/importance_sampling_ratio/min": 0.4127380847930908,
      "sampling/sampling_logp_difference/max": 0.8849420547485352,
      "sampling/sampling_logp_difference/mean": 0.01255122572183609,
      "step": 9,
      "step_time": 92.58362530700037
    },
    {
      "clip_ratio/high_max": 0.010624476184602827,
      "clip_ratio/high_mean": 0.006026733753969893,
      "clip_ratio/low_mean": 0.01009578033699654,
      "clip_ratio/low_min": 0.0042697106837295,
      "clip_ratio/region_mean": 0.01612251391634345,
      "entropy": 0.08229666040278971,
      "epoch": 0.0001,
      "grad_norm": 1.487949252128601,
      "kl": 0.037161100313824136,
      "learning_rate": 2.5714285714285714e-05,
      "loss": -0.3665,
      "step": 10,
      "step_time": 28.38116288400022
    },
    {
      "clip_ratio/high_max": 0.0049977844464592636,
      "clip_ratio/high_mean": 0.0024988922232296318,
      "clip_ratio/low_mean": 0.0011143402953166515,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036132325185462832,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9244.0,
      "completions/max_terminated_length": 9244.0,
      "completions/mean_length": 7200.8125,
      "completions/mean_terminated_length": 7200.8125,
      "completions/min_length": 297.0,
      "completions/min_terminated_length": 297.0,
      "entropy": 0.06809334014542401,
      "epoch": 0.00011,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.8031041622161865,
      "kl": 0.015580801184114534,
      "learning_rate": 2.857142857142857e-05,
      "loss": -0.3986,
      "num_tokens": 1613108.0,
      "reward": -0.4156220555305481,
      "reward_std": 0.5291995406150818,
      "rewards/rollout_reward_func/mean": -0.4156220555305481,
      "rewards/rollout_reward_func/std": 0.5197964906692505,
      "sampling/importance_sampling_ratio/max": 2.4048261642456055,
      "sampling/importance_sampling_ratio/mean": 0.9991490244865417,
      "sampling/importance_sampling_ratio/min": 0.2525537610054016,
      "sampling/sampling_logp_difference/max": 1.3761311769485474,
      "sampling/sampling_logp_difference/mean": 0.009278691373765469,
      "step": 11,
      "step_time": 91.42414895300033
    },
    {
      "clip_ratio/high_max": 0.00853408733382821,
      "clip_ratio/high_mean": 0.004985434468835592,
      "clip_ratio/low_mean": 0.006528367113787681,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.011513801524415612,
      "entropy": 0.06515359785407782,
      "epoch": 0.00012,
      "grad_norm": 0.9123954176902771,
      "kl": 0.05925005234166747,
      "learning_rate": 3.142857142857143e-05,
      "loss": -0.4041,
      "step": 12,
      "step_time": 27.261401052000338
    },
    {
      "clip_ratio/high_max": 0.004881663771811873,
      "clip_ratio/high_mean": 0.0027880541456397623,
      "clip_ratio/low_mean": 0.0037891828105784953,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0065772370144259185,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9011.0,
      "completions/max_terminated_length": 9011.0,
      "completions/mean_length": 7481.90625,
      "completions/mean_terminated_length": 7481.90625,
      "completions/min_length": 2417.0,
      "completions/min_terminated_length": 2417.0,
      "entropy": 0.06863200129009783,
      "epoch": 0.00013,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.003671407699585,
      "kl": 0.039789786795154214,
      "learning_rate": 3.428571428571429e-05,
      "loss": -0.203,
      "num_tokens": 1880242.0,
      "reward": -0.5097812414169312,
      "reward_std": 0.4184800386428833,
      "rewards/rollout_reward_func/mean": -0.5097812414169312,
      "rewards/rollout_reward_func/std": 0.4387035071849823,
      "sampling/importance_sampling_ratio/max": 3.0,
      "sampling/importance_sampling_ratio/mean": 1.0012085437774658,
      "sampling/importance_sampling_ratio/min": 0.21886371076107025,
      "sampling/sampling_logp_difference/max": 2.316737651824951,
      "sampling/sampling_logp_difference/mean": 0.01692187413573265,
      "step": 13,
      "step_time": 93.44268519900038
    },
    {
      "clip_ratio/high_max": 0.012604236777406186,
      "clip_ratio/high_mean": 0.006302118388703093,
      "clip_ratio/low_mean": 0.01540282418136485,
      "clip_ratio/low_min": 0.003511236049234867,
      "clip_ratio/region_mean": 0.021704942628275603,
      "entropy": 0.06422114372253418,
      "epoch": 0.00014,
      "grad_norm": 1.1214606761932373,
      "kl": 0.07548845067503862,
      "learning_rate": 3.7142857142857143e-05,
      "loss": -0.2136,
      "step": 14,
      "step_time": 26.921327204999443
    },
    {
      "clip_ratio/high_max": 0.003742929140571505,
      "clip_ratio/high_mean": 0.0022186867718119174,
      "clip_ratio/low_mean": 0.0041773931588977575,
      "clip_ratio/low_min": 0.0014451349852606654,
      "clip_ratio/region_mean": 0.0063960799016058445,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9200.0,
      "completions/max_terminated_length": 9200.0,
      "completions/mean_length": 7406.59375,
      "completions/mean_terminated_length": 7406.59375,
      "completions/min_length": 203.0,
      "completions/min_terminated_length": 203.0,
      "entropy": 0.046332313562743366,
      "epoch": 0.00015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3658106327056885,
      "kl": 0.08860734064364806,
      "learning_rate": 4e-05,
      "loss": 0.0669,
      "num_tokens": 2144669.0,
      "reward": -0.3528895974159241,
      "reward_std": 0.443051815032959,
      "rewards/rollout_reward_func/mean": -0.3528895974159241,
      "rewards/rollout_reward_func/std": 0.48428139090538025,
      "sampling/importance_sampling_ratio/max": 1.8482800722122192,
      "sampling/importance_sampling_ratio/mean": 0.9966132640838623,
      "sampling/importance_sampling_ratio/min": 0.118045873939991,
      "sampling/sampling_logp_difference/max": 2.1366820335388184,
      "sampling/sampling_logp_difference/mean": 0.016083233058452606,
      "step": 15,
      "step_time": 91.83712665100029
    },
    {
      "clip_ratio/high_max": 0.004916455829516053,
      "clip_ratio/high_mean": 0.0024582279147580266,
      "clip_ratio/low_mean": 0.008916221850086004,
      "clip_ratio/low_min": 0.0021388453897088766,
      "clip_ratio/region_mean": 0.011374449823051691,
      "entropy": 0.04555429570609704,
      "epoch": 0.00016,
      "grad_norm": 1.3114889860153198,
      "kl": 0.1314735203923192,
      "learning_rate": 4.2857142857142856e-05,
      "loss": 0.0618,
      "step": 16,
      "step_time": 27.011356363000004
    },
    {
      "clip_ratio/high_max": 0.008057098719291389,
      "clip_ratio/high_mean": 0.00437577159027569,
      "clip_ratio/low_mean": 0.004278601991245523,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.008654373581521213,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9228.0,
      "completions/max_terminated_length": 9228.0,
      "completions/mean_length": 7819.875,
      "completions/mean_terminated_length": 7819.875,
      "completions/min_length": 3760.0,
      "completions/min_terminated_length": 3760.0,
      "entropy": 0.07362106395885348,
      "epoch": 0.00017,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.2947819232940674,
      "kl": 0.13101027125958353,
      "learning_rate": 4.5714285714285716e-05,
      "loss": -0.0547,
      "num_tokens": 2422510.0,
      "reward": -0.35005444288253784,
      "reward_std": 0.49921366572380066,
      "rewards/rollout_reward_func/mean": -0.35005444288253784,
      "rewards/rollout_reward_func/std": 0.522201657295227,
      "sampling/importance_sampling_ratio/max": 3.0,
      "sampling/importance_sampling_ratio/mean": 1.0008654594421387,
      "sampling/importance_sampling_ratio/min": 0.04268655180931091,
      "sampling/sampling_logp_difference/max": 3.1538712978363037,
      "sampling/sampling_logp_difference/mean": 0.027018871158361435,
      "step": 17,
      "step_time": 96.36866129600003
    },
    {
      "clip_ratio/high_max": 0.011791770870331675,
      "clip_ratio/high_mean": 0.0066107546736020595,
      "clip_ratio/low_mean": 0.01208667631726712,
      "clip_ratio/low_min": 0.0028746770112775266,
      "clip_ratio/region_mean": 0.01869743096176535,
      "entropy": 0.0709565794095397,
      "epoch": 0.00018,
      "grad_norm": 3.9747724533081055,
      "kl": 0.45611770218238235,
      "learning_rate": 4.8571428571428576e-05,
      "loss": -0.0527,
      "step": 18,
      "step_time": 27.170892336000406
    },
    {
      "clip_ratio/high_max": 0.006368288770318031,
      "clip_ratio/high_mean": 0.0031841443851590157,
      "clip_ratio/low_mean": 0.0007183908019214869,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0039025351288728416,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9443.0,
      "completions/max_terminated_length": 9443.0,
      "completions/mean_length": 7551.3125,
      "completions/mean_terminated_length": 7551.3125,
      "completions/min_length": 2300.0,
      "completions/min_terminated_length": 2300.0,
      "entropy": 0.05112030985765159,
      "epoch": 0.00019,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3253093957901,
      "kl": 0.06368285935604945,
      "learning_rate": 5.142857142857143e-05,
      "loss": -0.1504,
      "num_tokens": 2691652.0,
      "reward": -0.4411429762840271,
      "reward_std": 0.4299367666244507,
      "rewards/rollout_reward_func/mean": -0.4411429762840271,
      "rewards/rollout_reward_func/std": 0.4561052918434143,
      "sampling/importance_sampling_ratio/max": 1.6581709384918213,
      "sampling/importance_sampling_ratio/mean": 0.997830867767334,
      "sampling/importance_sampling_ratio/min": 0.07465386390686035,
      "sampling/sampling_logp_difference/max": 2.594892978668213,
      "sampling/sampling_logp_difference/mean": 0.013442842289805412,
      "step": 19,
      "step_time": 93.42405358899964
    },
    {
      "clip_ratio/high_max": 0.01606024947250262,
      "clip_ratio/high_mean": 0.008740443765418604,
      "clip_ratio/low_mean": 0.006419806071789935,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.015160249720793217,
      "entropy": 0.053090699831955135,
      "epoch": 0.0002,
      "grad_norm": 1.2332507371902466,
      "kl": 0.1118333391204942,
      "learning_rate": 5.428571428571428e-05,
      "loss": -0.156,
      "step": 20,
      "step_time": 27.701954217999173
    },
    {
      "clip_ratio/high_max": 0.005651562649291009,
      "clip_ratio/high_mean": 0.0028257813246455044,
      "clip_ratio/low_mean": 0.0027939277642872185,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.005619709118036553,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9189.0,
      "completions/max_terminated_length": 9189.0,
      "completions/mean_length": 7761.09375,
      "completions/mean_terminated_length": 7761.09375,
      "completions/min_length": 1417.0,
      "completions/min_terminated_length": 1417.0,
      "entropy": 0.06741820118622854,
      "epoch": 0.00021,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 3.6862173080444336,
      "kl": 0.4830274658452254,
      "learning_rate": 5.714285714285714e-05,
      "loss": -0.2263,
      "num_tokens": 2967426.0,
      "reward": -0.28399354219436646,
      "reward_std": 0.4300430417060852,
      "rewards/rollout_reward_func/mean": -0.28399354219436646,
      "rewards/rollout_reward_func/std": 0.5347884297370911,
      "sampling/importance_sampling_ratio/max": 2.1812915802001953,
      "sampling/importance_sampling_ratio/mean": 0.997977614402771,
      "sampling/importance_sampling_ratio/min": 0.4058607518672943,
      "sampling/sampling_logp_difference/max": 0.9017452001571655,
      "sampling/sampling_logp_difference/mean": 0.014318476431071758,
      "step": 21,
      "step_time": 94.09382517800077
    },
    {
      "clip_ratio/high_max": 0.009377924376167357,
      "clip_ratio/high_mean": 0.006085653760237619,
      "clip_ratio/low_mean": 0.012686202622717246,
      "clip_ratio/low_min": 0.002905860950704664,
      "clip_ratio/region_mean": 0.018771856353851035,
      "entropy": 0.07064696005545557,
      "epoch": 0.00022,
      "grad_norm": 6.065097332000732,
      "kl": 0.08845078520243987,
      "learning_rate": 6e-05,
      "loss": -0.2227,
      "step": 22,
      "step_time": 27.40954218999991
    },
    {
      "clip_ratio/high_max": 0.0028251263429410756,
      "clip_ratio/high_mean": 0.0017597853729967028,
      "clip_ratio/low_mean": 0.0020223904284648597,
      "clip_ratio/low_min": 0.0005252101109363139,
      "clip_ratio/region_mean": 0.0037821758014615625,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8721.0,
      "completions/max_terminated_length": 8721.0,
      "completions/mean_length": 7136.59375,
      "completions/mean_terminated_length": 7136.59375,
      "completions/min_length": 959.0,
      "completions/min_terminated_length": 959.0,
      "entropy": 0.06426455150358379,
      "epoch": 0.00023,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 2.0993857383728027,
      "kl": 0.1524171840865165,
      "learning_rate": 6.285714285714286e-05,
      "loss": -0.0482,
      "num_tokens": 3222856.0,
      "reward": -0.365644633769989,
      "reward_std": 0.5254338979721069,
      "rewards/rollout_reward_func/mean": -0.365644633769989,
      "rewards/rollout_reward_func/std": 0.5246545076370239,
      "sampling/importance_sampling_ratio/max": 2.320268392562866,
      "sampling/importance_sampling_ratio/mean": 0.9996501803398132,
      "sampling/importance_sampling_ratio/min": 0.09908817708492279,
      "sampling/sampling_logp_difference/max": 2.3117451667785645,
      "sampling/sampling_logp_difference/mean": 0.011567876674234867,
      "step": 23,
      "step_time": 89.85609540400014
    },
    {
      "clip_ratio/high_max": 0.009038183547090739,
      "clip_ratio/high_mean": 0.005584432714385912,
      "clip_ratio/low_mean": 0.005201028659939766,
      "clip_ratio/low_min": 0.0015668768319301307,
      "clip_ratio/region_mean": 0.010785461287014186,
      "entropy": 0.06729507446289062,
      "epoch": 0.00024,
      "grad_norm": 1.5627487897872925,
      "kl": 0.1648054391844198,
      "learning_rate": 6.571428571428571e-05,
      "loss": -0.0548,
      "step": 24,
      "step_time": 25.88749120899979
    },
    {
      "clip_ratio/high_max": 0.005618693074211478,
      "clip_ratio/high_mean": 0.002809346537105739,
      "clip_ratio/low_mean": 0.001929406455019489,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004738752992125228,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8856.0,
      "completions/max_terminated_length": 8856.0,
      "completions/mean_length": 7467.65625,
      "completions/mean_terminated_length": 7467.65625,
      "completions/min_length": 402.0,
      "completions/min_terminated_length": 402.0,
      "entropy": 0.07340177637524903,
      "epoch": 0.00025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2772353887557983,
      "kl": 0.0859682586742565,
      "learning_rate": 6.857142857142858e-05,
      "loss": -0.2314,
      "num_tokens": 3488411.0,
      "reward": -0.3830329477787018,
      "reward_std": 0.5107574462890625,
      "rewards/rollout_reward_func/mean": -0.3830329477787018,
      "rewards/rollout_reward_func/std": 0.5173285007476807,
      "sampling/importance_sampling_ratio/max": 2.1774182319641113,
      "sampling/importance_sampling_ratio/mean": 0.999387264251709,
      "sampling/importance_sampling_ratio/min": 0.29978206753730774,
      "sampling/sampling_logp_difference/max": 1.2046995162963867,
      "sampling/sampling_logp_difference/mean": 0.011983634904026985,
      "step": 25,
      "step_time": 92.61141428400015
    },
    {
      "clip_ratio/high_max": 0.014493243885226548,
      "clip_ratio/high_mean": 0.008288288547191769,
      "clip_ratio/low_mean": 0.013486813055351377,
      "clip_ratio/low_min": 0.0028249489841982722,
      "clip_ratio/region_mean": 0.021775101457023993,
      "entropy": 0.0781157671008259,
      "epoch": 0.00026,
      "grad_norm": 0.8266052603721619,
      "kl": 0.08862257207510993,
      "learning_rate": 7.142857142857143e-05,
      "loss": -0.2407,
      "step": 26,
      "step_time": 26.541104049000296
    },
    {
      "clip_ratio/high_max": 0.0022112573496997356,
      "clip_ratio/high_mean": 0.001456752244848758,
      "clip_ratio/low_mean": 0.0013966916594654322,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00285344390431419,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9222.0,
      "completions/max_terminated_length": 9222.0,
      "completions/mean_length": 7990.125,
      "completions/mean_terminated_length": 7990.125,
      "completions/min_length": 2192.0,
      "completions/min_terminated_length": 2192.0,
      "entropy": 0.08974491525441408,
      "epoch": 0.00027,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0755014419555664,
      "kl": 0.08955412585055456,
      "learning_rate": 7.428571428571429e-05,
      "loss": -0.0612,
      "num_tokens": 3770720.0,
      "reward": -0.1771392822265625,
      "reward_std": 0.4869571924209595,
      "rewards/rollout_reward_func/mean": -0.1771392822265625,
      "rewards/rollout_reward_func/std": 0.46993595361709595,
      "sampling/importance_sampling_ratio/max": 1.7862199544906616,
      "sampling/importance_sampling_ratio/mean": 1.0007758140563965,
      "sampling/importance_sampling_ratio/min": 0.294488787651062,
      "sampling/sampling_logp_difference/max": 1.2225143909454346,
      "sampling/sampling_logp_difference/mean": 0.011052686721086502,
      "step": 27,
      "step_time": 95.3786270669998
    },
    {
      "clip_ratio/high_max": 0.012634392420295626,
      "clip_ratio/high_mean": 0.00806891280808486,
      "clip_ratio/low_mean": 0.005300049553625286,
      "clip_ratio/low_min": 0.0014046717551536858,
      "clip_ratio/region_mean": 0.013368962419917807,
      "entropy": 0.09402668941766024,
      "epoch": 0.00028,
      "grad_norm": 0.9924318790435791,
      "kl": 0.0753595522546675,
      "learning_rate": 7.714285714285715e-05,
      "loss": -0.0693,
      "step": 28,
      "step_time": 27.647992973000328
    },
    {
      "clip_ratio/high_max": 0.0035870341234840453,
      "clip_ratio/high_mean": 0.002140739292372018,
      "clip_ratio/low_mean": 0.0017440025694668293,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003884741832735017,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9067.0,
      "completions/max_terminated_length": 9067.0,
      "completions/mean_length": 7836.9375,
      "completions/mean_terminated_length": 7836.9375,
      "completions/min_length": 1928.0,
      "completions/min_terminated_length": 1928.0,
      "entropy": 0.09281386900693178,
      "epoch": 0.00029,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.068334937095642,
      "kl": 0.0688294384162873,
      "learning_rate": 8e-05,
      "loss": 0.0442,
      "num_tokens": 4047979.0,
      "reward": -0.25341448187828064,
      "reward_std": 0.49919599294662476,
      "rewards/rollout_reward_func/mean": -0.25341448187828064,
      "rewards/rollout_reward_func/std": 0.500785231590271,
      "sampling/importance_sampling_ratio/max": 1.868089199066162,
      "sampling/importance_sampling_ratio/mean": 1.0002467632293701,
      "sampling/importance_sampling_ratio/min": 0.5016597509384155,
      "sampling/sampling_logp_difference/max": 0.6898331642150879,
      "sampling/sampling_logp_difference/mean": 0.009339751675724983,
      "step": 29,
      "step_time": 94.94639882799993
    },
    {
      "clip_ratio/high_max": 0.015361635130830109,
      "clip_ratio/high_mean": 0.0076808175654150546,
      "clip_ratio/low_mean": 0.007387197489151731,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.015068015054566786,
      "entropy": 0.09539854992181063,
      "epoch": 0.0003,
      "grad_norm": 0.7568414807319641,
      "kl": 0.06834399094805121,
      "learning_rate": 8.285714285714287e-05,
      "loss": 0.0376,
      "step": 30,
      "step_time": 26.633041288999266
    },
    {
      "clip_ratio/high_max": 0.004851312725804746,
      "clip_ratio/high_mean": 0.002425656362902373,
      "clip_ratio/low_mean": 0.003376906446646899,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.00580256269313395,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9536.0,
      "completions/max_terminated_length": 9536.0,
      "completions/mean_length": 8069.3125,
      "completions/mean_terminated_length": 8069.3125,
      "completions/min_length": 2100.0,
      "completions/min_terminated_length": 2100.0,
      "entropy": 0.1009219279512763,
      "epoch": 0.00031,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0075691938400269,
      "kl": 0.06613797519821674,
      "learning_rate": 8.571428571428571e-05,
      "loss": 0.0875,
      "num_tokens": 4332991.0,
      "reward": -0.2427319884300232,
      "reward_std": 0.5179601907730103,
      "rewards/rollout_reward_func/mean": -0.2427319884300232,
      "rewards/rollout_reward_func/std": 0.5765778422355652,
      "sampling/importance_sampling_ratio/max": 1.7835460901260376,
      "sampling/importance_sampling_ratio/mean": 0.9999290704727173,
      "sampling/importance_sampling_ratio/min": 0.6331738829612732,
      "sampling/sampling_logp_difference/max": 0.5786035060882568,
      "sampling/sampling_logp_difference/mean": 0.009435810148715973,
      "step": 31,
      "step_time": 94.81605570200009
    },
    {
      "clip_ratio/high_max": 0.017235137231182307,
      "clip_ratio/high_mean": 0.008985215710708871,
      "clip_ratio/low_mean": 0.012812130036763847,
      "clip_ratio/low_min": 0.0027777778450399637,
      "clip_ratio/region_mean": 0.021797345718368888,
      "entropy": 0.10036895563825965,
      "epoch": 0.00032,
      "grad_norm": 0.7584834098815918,
      "kl": 0.11454511061310768,
      "learning_rate": 8.857142857142857e-05,
      "loss": 0.0788,
      "step": 32,
      "step_time": 28.761855419000312
    },
    {
      "clip_ratio/high_max": 0.0031494522700086236,
      "clip_ratio/high_mean": 0.0015747261350043118,
      "clip_ratio/low_mean": 0.0024462497676722705,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004020975931780413,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9354.0,
      "completions/max_terminated_length": 9354.0,
      "completions/mean_length": 7928.5,
      "completions/mean_terminated_length": 7928.5,
      "completions/min_length": 3150.0,
      "completions/min_terminated_length": 3150.0,
      "entropy": 0.1094990405254066,
      "epoch": 0.00033,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8852567672729492,
      "kl": 0.0793593015987426,
      "learning_rate": 9.142857142857143e-05,
      "loss": 0.1924,
      "num_tokens": 4613150.0,
      "reward": -0.1065119206905365,
      "reward_std": 0.6632397174835205,
      "rewards/rollout_reward_func/mean": -0.1065119206905365,
      "rewards/rollout_reward_func/std": 0.6485887765884399,
      "sampling/importance_sampling_ratio/max": 1.593437671661377,
      "sampling/importance_sampling_ratio/mean": 0.9991821050643921,
      "sampling/importance_sampling_ratio/min": 0.5760508179664612,
      "sampling/sampling_logp_difference/max": 0.551559329032898,
      "sampling/sampling_logp_difference/mean": 0.0090299928560853,
      "step": 33,
      "step_time": 94.56970139799978
    },
    {
      "clip_ratio/high_max": 0.010831523861270398,
      "clip_ratio/high_mean": 0.006442332087317482,
      "clip_ratio/low_mean": 0.014884777745464817,
      "clip_ratio/low_min": 0.004198966489639133,
      "clip_ratio/region_mean": 0.02132710992009379,
      "entropy": 0.1035804757848382,
      "epoch": 0.00034,
      "grad_norm": 6.932408332824707,
      "kl": 0.36925772204995155,
      "learning_rate": 9.428571428571429e-05,
      "loss": 0.19,
      "step": 34,
      "step_time": 28.189770212999974
    },
    {
      "clip_ratio/high_max": 0.004503837670199573,
      "clip_ratio/high_mean": 0.0022519188350997865,
      "clip_ratio/low_mean": 0.001957070722710341,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.004208989586913958,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9253.0,
      "completions/max_terminated_length": 9253.0,
      "completions/mean_length": 8121.40625,
      "completions/mean_terminated_length": 8121.40625,
      "completions/min_length": 3211.0,
      "completions/min_terminated_length": 3211.0,
      "entropy": 0.08986494969576597,
      "epoch": 0.00035,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.0189476013183594,
      "kl": 0.11070865276269615,
      "learning_rate": 9.714285714285715e-05,
      "loss": -0.1358,
      "num_tokens": 4899944.0,
      "reward": -0.3683607578277588,
      "reward_std": 0.39094337821006775,
      "rewards/rollout_reward_func/mean": -0.3683607578277588,
      "rewards/rollout_reward_func/std": 0.3886735141277313,
      "sampling/importance_sampling_ratio/max": 1.678568720817566,
      "sampling/importance_sampling_ratio/mean": 0.9991096258163452,
      "sampling/importance_sampling_ratio/min": 0.2759600579738617,
      "sampling/sampling_logp_difference/max": 1.287499189376831,
      "sampling/sampling_logp_difference/mean": 0.009727813303470612,
      "step": 35,
      "step_time": 95.49759846299912
    },
    {
      "clip_ratio/high_max": 0.016388497198931873,
      "clip_ratio/high_mean": 0.008888693118933588,
      "clip_ratio/low_mean": 0.012551680905744433,
      "clip_ratio/low_min": 0.002083333383779973,
      "clip_ratio/region_mean": 0.021440374111989513,
      "entropy": 0.08508480247110128,
      "epoch": 0.00036,
      "grad_norm": 0.7093645334243774,
      "kl": 0.13321141549386084,
      "learning_rate": 0.0001,
      "loss": -0.142,
      "step": 36,
      "step_time": 28.9255163910002
    },
    {
      "clip_ratio/high_max": 0.0037438530707731843,
      "clip_ratio/high_mean": 0.0018719265353865921,
      "clip_ratio/low_mean": 0.0017557953542564064,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0036277218896429986,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8904.0,
      "completions/max_terminated_length": 8904.0,
      "completions/mean_length": 7886.75,
      "completions/mean_terminated_length": 7886.75,
      "completions/min_length": 2730.0,
      "completions/min_terminated_length": 2730.0,
      "entropy": 0.0856572980992496,
      "epoch": 0.00037,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8132036328315735,
      "kl": 0.10761717008426785,
      "learning_rate": 9.918053502751771e-05,
      "loss": -0.0071,
      "num_tokens": 5179039.0,
      "reward": -0.2326236367225647,
      "reward_std": 0.5471592545509338,
      "rewards/rollout_reward_func/mean": -0.2326236367225647,
      "rewards/rollout_reward_func/std": 0.5504636168479919,
      "sampling/importance_sampling_ratio/max": 1.7522045373916626,
      "sampling/importance_sampling_ratio/mean": 0.9999704360961914,
      "sampling/importance_sampling_ratio/min": 0.6301739811897278,
      "sampling/sampling_logp_difference/max": 0.5608747005462646,
      "sampling/sampling_logp_difference/mean": 0.00934191420674324,
      "step": 37,
      "step_time": 91.66157414699956
    },
    {
      "clip_ratio/high_max": 0.009254805103410035,
      "clip_ratio/high_mean": 0.004974624811438844,
      "clip_ratio/low_mean": 0.0146892913908232,
      "clip_ratio/low_min": 0.002091136120725423,
      "clip_ratio/region_mean": 0.019663916405988857,
      "entropy": 0.08511333051137626,
      "epoch": 0.00038,
      "grad_norm": 1.016090750694275,
      "kl": 0.13888527592644095,
      "learning_rate": 9.675795466159755e-05,
      "loss": -0.0123,
      "step": 38,
      "step_time": 27.996872589999384
    },
    {
      "clip_ratio/high_max": 0.004166666767559946,
      "clip_ratio/high_mean": 0.002083333383779973,
      "clip_ratio/low_mean": 0.003956988512072712,
      "clip_ratio/low_min": 0.0013966916594654322,
      "clip_ratio/region_mean": 0.006040321895852685,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9174.0,
      "completions/max_terminated_length": 9174.0,
      "completions/mean_length": 7773.78125,
      "completions/mean_terminated_length": 7773.78125,
      "completions/min_length": 644.0,
      "completions/min_terminated_length": 644.0,
      "entropy": 0.08465631771832705,
      "epoch": 0.00039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2169103622436523,
      "kl": 0.08935846015810966,
      "learning_rate": 9.283813728906054e-05,
      "loss": -0.1228,
      "num_tokens": 5455087.0,
      "reward": -0.23303870856761932,
      "reward_std": 0.587963879108429,
      "rewards/rollout_reward_func/mean": -0.23303870856761932,
      "rewards/rollout_reward_func/std": 0.6264134645462036,
      "sampling/importance_sampling_ratio/max": 2.5134615898132324,
      "sampling/importance_sampling_ratio/mean": 1.000549077987671,
      "sampling/importance_sampling_ratio/min": 0.555368959903717,
      "sampling/sampling_logp_difference/max": 0.9216609001159668,
      "sampling/sampling_logp_difference/mean": 0.009383916854858398,
      "step": 39,
      "step_time": 87.98878976900005
    },
    {
      "clip_ratio/high_max": 0.007900029362645,
      "clip_ratio/high_mean": 0.004326520691392943,
      "clip_ratio/low_mean": 0.011018106888514012,
      "clip_ratio/low_min": 0.0018596546142362058,
      "clip_ratio/region_mean": 0.015344627550803125,
      "entropy": 0.08598045213147998,
      "epoch": 0.0004,
      "grad_norm": 1.855177640914917,
      "kl": 0.08451172313652933,
      "learning_rate": 8.759239773845718e-05,
      "loss": -0.1252,
      "step": 40,
      "step_time": 28.765726263999568
    },
    {
      "clip_ratio/high_max": 0.005807758076116443,
      "clip_ratio/high_mean": 0.003251101268688217,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003251101268688217,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8897.0,
      "completions/max_terminated_length": 8897.0,
      "completions/mean_length": 7154.53125,
      "completions/mean_terminated_length": 7154.53125,
      "completions/min_length": 2028.0,
      "completions/min_terminated_length": 2028.0,
      "entropy": 0.07370180264115334,
      "epoch": 0.00041,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6197995543479919,
      "kl": 0.08875341480597854,
      "learning_rate": 8.125000000000001e-05,
      "loss": -0.0075,
      "num_tokens": 5710693.0,
      "reward": -0.11052435636520386,
      "reward_std": 0.6305065155029297,
      "rewards/rollout_reward_func/mean": -0.11052435636520386,
      "rewards/rollout_reward_func/std": 0.6378242373466492,
      "sampling/importance_sampling_ratio/max": 1.3583569526672363,
      "sampling/importance_sampling_ratio/mean": 0.9987045526504517,
      "sampling/importance_sampling_ratio/min": 0.3902639150619507,
      "sampling/sampling_logp_difference/max": 0.940932035446167,
      "sampling/sampling_logp_difference/mean": 0.007857292890548706,
      "step": 41,
      "step_time": 89.4707287680003
    },
    {
      "clip_ratio/high_max": 0.011104670935310423,
      "clip_ratio/high_mean": 0.005552335467655212,
      "clip_ratio/low_mean": 0.0056608943559695035,
      "clip_ratio/low_min": 0.0024853801587596536,
      "clip_ratio/region_mean": 0.011213229881832376,
      "entropy": 0.07147298031486571,
      "epoch": 0.00042,
      "grad_norm": 0.532979428768158,
      "kl": 0.1001066651660949,
      "learning_rate": 7.408813728906053e-05,
      "loss": -0.0097,
      "step": 42,
      "step_time": 27.083737006999854
    },
    {
      "clip_ratio/high_max": 0.002190170984249562,
      "clip_ratio/high_mean": 0.001095085492124781,
      "clip_ratio/low_mean": 0.0017723122437018901,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002867397735826671,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9047.0,
      "completions/max_terminated_length": 9047.0,
      "completions/mean_length": 7559.5625,
      "completions/mean_terminated_length": 7559.5625,
      "completions/min_length": 714.0,
      "completions/min_terminated_length": 714.0,
      "entropy": 0.07850536378100514,
      "epoch": 0.00043,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6568584442138672,
      "kl": 0.13246144936420023,
      "learning_rate": 6.6419817372537e-05,
      "loss": -0.164,
      "num_tokens": 5979343.0,
      "reward": -0.18616318702697754,
      "reward_std": 0.5167393088340759,
      "rewards/rollout_reward_func/mean": -0.18616318702697754,
      "rewards/rollout_reward_func/std": 0.5579463243484497,
      "sampling/importance_sampling_ratio/max": 1.8209000825881958,
      "sampling/importance_sampling_ratio/mean": 1.0000110864639282,
      "sampling/importance_sampling_ratio/min": 0.5961325764656067,
      "sampling/sampling_logp_difference/max": 0.5993309020996094,
      "sampling/sampling_logp_difference/mean": 0.007760843262076378,
      "step": 43,
      "step_time": 89.20096494900008
    },
    {
      "clip_ratio/high_max": 0.011211407894734293,
      "clip_ratio/high_mean": 0.0063001484086271375,
      "clip_ratio/low_mean": 0.00652579654706642,
      "clip_ratio/low_min": 0.0014211886445991695,
      "clip_ratio/region_mean": 0.012825944955693558,
      "entropy": 0.07914698123931885,
      "epoch": 0.00044,
      "grad_norm": 0.4922528564929962,
      "kl": 0.1358487531542778,
      "learning_rate": 5.858018262746301e-05,
      "loss": -0.1705,
      "step": 44,
      "step_time": 28.31045281400202
    },
    {
      "clip_ratio/high_max": 0.0007102272938936949,
      "clip_ratio/high_mean": 0.00035511364694684744,
      "clip_ratio/low_mean": 0.0017025089473463595,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.002057622594293207,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 8960.0,
      "completions/max_terminated_length": 8960.0,
      "completions/mean_length": 7259.9375,
      "completions/mean_terminated_length": 7259.9375,
      "completions/min_length": 1793.0,
      "completions/min_terminated_length": 1793.0,
      "entropy": 0.0675961917731911,
      "epoch": 0.00045,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4965215027332306,
      "kl": 0.12192867766134441,
      "learning_rate": 5.0911862710939485e-05,
      "loss": 0.0928,
      "num_tokens": 6238685.0,
      "reward": -0.19794124364852905,
      "reward_std": 0.6908308267593384,
      "rewards/rollout_reward_func/mean": -0.19794124364852905,
      "rewards/rollout_reward_func/std": 0.6924669742584229,
      "sampling/importance_sampling_ratio/max": 1.7397383451461792,
      "sampling/importance_sampling_ratio/mean": 1.0005090236663818,
      "sampling/importance_sampling_ratio/min": 0.6195511817932129,
      "sampling/sampling_logp_difference/max": 0.5537347793579102,
      "sampling/sampling_logp_difference/mean": 0.006288852542638779,
      "step": 45,
      "step_time": 88.08988413000043
    },
    {
      "clip_ratio/high_max": 0.0034880051389336586,
      "clip_ratio/high_mean": 0.0017440025694668293,
      "clip_ratio/low_mean": 0.0030215850565582514,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.004765587626025081,
      "entropy": 0.06737200380302966,
      "epoch": 0.00046,
      "grad_norm": 0.537150502204895,
      "kl": 0.12145327543839812,
      "learning_rate": 4.375000000000001e-05,
      "loss": 0.092,
      "step": 46,
      "step_time": 27.73081736800077
    },
    {
      "clip_ratio/high_max": 0.001753766497131437,
      "clip_ratio/high_mean": 0.0012241055082995445,
      "clip_ratio/low_mean": 0.001392790290992707,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0026168957992922515,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9253.0,
      "completions/max_terminated_length": 9253.0,
      "completions/mean_length": 7692.15625,
      "completions/mean_terminated_length": 7692.15625,
      "completions/min_length": 293.0,
      "completions/min_terminated_length": 293.0,
      "entropy": 0.0828928763512522,
      "epoch": 0.00047,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5421693921089172,
      "kl": 0.13829960371367633,
      "learning_rate": 3.740760226154283e-05,
      "loss": 0.1124,
      "num_tokens": 6512018.0,
      "reward": -0.13337820768356323,
      "reward_std": 0.6952483057975769,
      "rewards/rollout_reward_func/mean": -0.13337820768356323,
      "rewards/rollout_reward_func/std": 0.6867809295654297,
      "sampling/importance_sampling_ratio/max": 1.9118043184280396,
      "sampling/importance_sampling_ratio/mean": 1.000080943107605,
      "sampling/importance_sampling_ratio/min": 0.6484359502792358,
      "sampling/sampling_logp_difference/max": 0.6480474472045898,
      "sampling/sampling_logp_difference/mean": 0.007844729349017143,
      "step": 47,
      "step_time": 89.50516965200086
    },
    {
      "clip_ratio/high_max": 0.0034878277801908553,
      "clip_ratio/high_mean": 0.002519218367524445,
      "clip_ratio/low_mean": 0.00764669178170152,
      "clip_ratio/low_min": 0.0013888889225199819,
      "clip_ratio/region_mean": 0.010165910149225965,
      "entropy": 0.08108305186033249,
      "epoch": 0.00048,
      "grad_norm": 0.5435065031051636,
      "kl": 0.15144605189561844,
      "learning_rate": 3.2161862710939476e-05,
      "loss": 0.1097,
      "step": 48,
      "step_time": 28.342568254000525
    },
    {
      "clip_ratio/high_max": 0.004411048837937415,
      "clip_ratio/high_mean": 0.0022055244189687073,
      "clip_ratio/low_mean": 0.0010455680603627115,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.003251092479331419,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 9355.0,
      "completions/max_terminated_length": 9355.0,
      "completions/mean_length": 7641.09375,
      "completions/mean_terminated_length": 7641.09375,
      "completions/min_length": 395.0,
      "completions/min_terminated_length": 395.0,
      "entropy": 0.06896222359500825,
      "epoch": 0.00049,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7906138300895691,
      "kl": 0.17947973730042577,
      "learning_rate": 2.8242045338402468e-05,
      "loss": 0.0025,
      "num_tokens": 6783110.0,
      "reward": -0.24048486351966858,
      "reward_std": 0.5240083932876587,
      "rewards/rollout_reward_func/mean": -0.24048486351966858,
      "rewards/rollout_reward_func/std": 0.6190164685249329,
      "sampling/importance_sampling_ratio/max": 1.339056372642517,
      "sampling/importance_sampling_ratio/mean": 1.000028371810913,
      "sampling/importance_sampling_ratio/min": 0.6448177099227905,
      "sampling/sampling_logp_difference/max": 0.43878769874572754,
      "sampling/sampling_logp_difference/mean": 0.0061599984765052795,
      "step": 49,
      "step_time": 91.6421234169984
    },
    {
      "clip_ratio/high_max": 0.00502964504994452,
      "clip_ratio/high_mean": 0.00251482252497226,
      "clip_ratio/low_mean": 0.00382724724477157,
      "clip_ratio/low_min": 0.0006944444612599909,
      "clip_ratio/region_mean": 0.00634206976974383,
      "entropy": 0.0660756464349106,
      "epoch": 0.0005,
      "grad_norm": 0.6092560887336731,
      "kl": 0.1758871516212821,
      "learning_rate": 2.5819464972482288e-05,
      "loss": -0.0001,
      "step": 50,
      "step_time": 28.18612482699973
    }
  ],
  "logging_steps": 1.0,
  "max_steps": 50,
  "num_input_tokens_seen": 6783110,
  "num_train_epochs": 1,
  "save_steps": 500,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 1,
  "trial_name": null,
  "trial_params": null
}