v2critic/checkpoint-417/trainer_state.json

{
  "best_global_step": null,
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 0.06255625562556255,
  "eval_steps": 100,
  "global_step": 417,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 813.4,
      "completions/max_terminated_length": 813.4,
      "completions/mean_length": 732.3125244140625,
      "completions/mean_terminated_length": 732.3125244140625,
      "completions/min_length": 547.6,
      "completions/min_terminated_length": 547.6,
      "entropy": 0.6945622801780701,
      "epoch": 0.00075007500750075,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.71484375,
      "learning_rate": 9.904076738609112e-07,
      "loss": -0.0662,
      "num_tokens": 556763.0,
      "reward": 0.25436362624168396,
      "reward_std": 0.22843295931816102,
      "rewards/ADEnReward/mean": 0.03650980964303017,
      "rewards/ADEnReward/std": 0.0942004919052124,
      "rewards/CriticV2Reward/mean": 0.12452046498656273,
      "rewards/CriticV2Reward/std": 0.2022954449057579,
      "rewards/StrictFormatReward/mean": 0.9333333373069763,
      "rewards/StrictFormatReward/std": 0.3171369910240173,
      "sampling/importance_sampling_ratio/max": 2.736471652984619,
      "sampling/importance_sampling_ratio/mean": 0.42077494263648985,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.499092960357666,
      "sampling/sampling_logp_difference/mean": 0.027753319963812827,
      "step": 5,
      "step_time": 16.534613192547113
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 803.0,
      "completions/max_terminated_length": 803.0,
      "completions/mean_length": 733.6416870117188,
      "completions/mean_terminated_length": 733.6416870117188,
      "completions/min_length": 630.0,
      "completions/min_terminated_length": 630.0,
      "entropy": 0.6582455694675445,
      "epoch": 0.0015001500150015,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.3203125,
      "learning_rate": 9.784172661870503e-07,
      "loss": -0.0192,
      "num_tokens": 1114181.0,
      "reward": 0.2501808494329453,
      "reward_std": 0.22661161720752715,
      "rewards/ADEnReward/mean": 0.01843893507029861,
      "rewards/ADEnReward/std": 0.04499729741364718,
      "rewards/CriticV2Reward/mean": 0.13840857446193694,
      "rewards/CriticV2Reward/std": 0.21599454879760743,
      "rewards/StrictFormatReward/mean": 0.9333333373069763,
      "rewards/StrictFormatReward/std": 0.3171369910240173,
      "sampling/importance_sampling_ratio/max": 2.801686000823975,
      "sampling/importance_sampling_ratio/mean": 0.4209885835647583,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.701645183563232,
      "sampling/sampling_logp_difference/mean": 0.027708011493086816,
      "step": 10,
      "step_time": 13.271006786078214
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 963.6,
      "completions/max_terminated_length": 963.6,
      "completions/mean_length": 738.40419921875,
      "completions/mean_terminated_length": 738.40419921875,
      "completions/min_length": 645.2,
      "completions/min_terminated_length": 645.2,
      "entropy": 0.6592447817325592,
      "epoch": 0.0022502250225022503,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8359375,
      "learning_rate": 9.664268585131893e-07,
      "loss": 0.0102,
      "num_tokens": 1672550.0,
      "reward": 0.3058495342731476,
      "reward_std": 0.2918362975120544,
      "rewards/ADEnReward/mean": 0.04017725773155689,
      "rewards/ADEnReward/std": 0.09973704516887665,
      "rewards/CriticV2Reward/mean": 0.1756722643971443,
      "rewards/CriticV2Reward/std": 0.2586840629577637,
      "rewards/StrictFormatReward/mean": 0.9000000119209289,
      "rewards/StrictFormatReward/std": 0.434829568862915,
      "sampling/importance_sampling_ratio/max": 2.4545116901397703,
      "sampling/importance_sampling_ratio/mean": 0.37839311361312866,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.780246067047119,
      "sampling/sampling_logp_difference/mean": 0.027404751628637314,
      "step": 15,
      "step_time": 14.447486751060932
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 826.6,
      "completions/max_terminated_length": 826.6,
      "completions/mean_length": 734.4125244140625,
      "completions/mean_terminated_length": 734.4125244140625,
      "completions/min_length": 583.4,
      "completions/min_terminated_length": 583.4,
      "entropy": 0.6629917621612549,
      "epoch": 0.003000300030003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3359375,
      "learning_rate": 9.544364508393285e-07,
      "loss": -0.0869,
      "num_tokens": 2230217.0,
      "reward": 0.29596259295940397,
      "reward_std": 0.2525408238172531,
      "rewards/ADEnReward/mean": 0.03129123505204916,
      "rewards/ADEnReward/std": 0.07233726158738137,
      "rewards/CriticV2Reward/mean": 0.16800468116998674,
      "rewards/CriticV2Reward/std": 0.23792918026447296,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.19624642729759217,
      "sampling/importance_sampling_ratio/max": 2.1899476528167723,
      "sampling/importance_sampling_ratio/mean": 0.3364464044570923,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.7939449310302735,
      "sampling/sampling_logp_difference/mean": 0.0276542279869318,
      "step": 20,
      "step_time": 13.247872665338218
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 927.4,
      "completions/max_terminated_length": 927.4,
      "completions/mean_length": 733.54169921875,
      "completions/mean_terminated_length": 733.54169921875,
      "completions/min_length": 668.4,
      "completions/min_terminated_length": 668.4,
      "entropy": 0.6701843202114105,
      "epoch": 0.0037503750375037503,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.65625,
      "learning_rate": 9.424460431654676e-07,
      "loss": -0.0358,
      "num_tokens": 2787211.0,
      "reward": 0.33442996740341185,
      "reward_std": 0.30991793870925904,
      "rewards/ADEnReward/mean": 0.055364159494638444,
      "rewards/ADEnReward/std": 0.11989772319793701,
      "rewards/CriticV2Reward/mean": 0.1823991283774376,
      "rewards/CriticV2Reward/std": 0.26879840195178983,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.1555842399597168,
      "sampling/importance_sampling_ratio/max": 2.724516677856445,
      "sampling/importance_sampling_ratio/mean": 0.4627986252307892,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.788603591918945,
      "sampling/sampling_logp_difference/mean": 0.026526946946978568,
      "step": 25,
      "step_time": 13.950103689264505
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 855.2,
      "completions/max_terminated_length": 855.2,
      "completions/mean_length": 736.0041870117187,
      "completions/mean_terminated_length": 736.0041870117187,
      "completions/min_length": 653.4,
      "completions/min_terminated_length": 653.4,
      "entropy": 0.6765582263469696,
      "epoch": 0.004500450045004501,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.53125,
      "learning_rate": 9.304556354916066e-07,
      "loss": -0.0296,
      "num_tokens": 3344860.0,
      "reward": 0.3146974563598633,
      "reward_std": 0.2686339944601059,
      "rewards/ADEnReward/mean": 0.02965494096279144,
      "rewards/ADEnReward/std": 0.07100159302353859,
      "rewards/CriticV2Reward/mean": 0.19254249930381775,
      "rewards/CriticV2Reward/std": 0.2526032477617264,
      "rewards/StrictFormatReward/mean": 0.924999988079071,
      "rewards/StrictFormatReward/std": 0.3250433325767517,
      "sampling/importance_sampling_ratio/max": 2.5650532245635986,
      "sampling/importance_sampling_ratio/mean": 0.410943877696991,
      "sampling/importance_sampling_ratio/min": 0.00010549233993515373,
      "sampling/sampling_logp_difference/max": 4.272568225860596,
      "sampling/sampling_logp_difference/mean": 0.027386844530701636,
      "step": 30,
      "step_time": 13.507405268307775
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 962.8,
      "completions/max_terminated_length": 815.8,
      "completions/mean_length": 740.1208618164062,
      "completions/mean_terminated_length": 736.7608276367188,
      "completions/min_length": 669.6,
      "completions/min_terminated_length": 669.6,
      "entropy": 0.6634912610054016,
      "epoch": 0.005250525052505251,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.447265625,
      "learning_rate": 9.184652278177458e-07,
      "loss": 0.0304,
      "num_tokens": 3903929.0,
      "reward": 0.284199857711792,
      "reward_std": 0.2560150265693665,
      "rewards/ADEnReward/mean": 0.02072476325556636,
      "rewards/ADEnReward/std": 0.053264892846345904,
      "rewards/CriticV2Reward/mean": 0.1701417475938797,
      "rewards/CriticV2Reward/std": 0.24270081222057344,
      "rewards/StrictFormatReward/mean": 0.9333333373069763,
      "rewards/StrictFormatReward/std": 0.3171369910240173,
      "sampling/importance_sampling_ratio/max": 2.2670124769210815,
      "sampling/importance_sampling_ratio/mean": 0.3484788566827774,
      "sampling/importance_sampling_ratio/min": 4.5519565901486205e-06,
      "sampling/sampling_logp_difference/max": 4.598708057403565,
      "sampling/sampling_logp_difference/mean": 0.02718786969780922,
      "step": 35,
      "step_time": 14.319740631990134
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 933.0,
      "completions/max_terminated_length": 933.0,
      "completions/mean_length": 741.645849609375,
      "completions/mean_terminated_length": 741.645849609375,
      "completions/min_length": 667.4,
      "completions/min_terminated_length": 667.4,
      "entropy": 0.6647457540035248,
      "epoch": 0.006000600060006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.84765625,
      "learning_rate": 9.064748201438849e-07,
      "loss": -0.0366,
      "num_tokens": 4463124.0,
      "reward": 0.29417389035224917,
      "reward_std": 0.26714652180671694,
      "rewards/ADEnReward/mean": 0.03201424721628428,
      "rewards/ADEnReward/std": 0.08157214596867561,
      "rewards/CriticV2Reward/mean": 0.16799295991659163,
      "rewards/CriticV2Reward/std": 0.246389901638031,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.294095641374588,
      "sampling/importance_sampling_ratio/max": 2.2125140190124513,
      "sampling/importance_sampling_ratio/mean": 0.35879886746406553,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.105627155303955,
      "sampling/sampling_logp_difference/mean": 0.027599729597568512,
      "step": 40,
      "step_time": 14.012726863101125
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1093.6,
      "completions/max_terminated_length": 939.8,
      "completions/mean_length": 735.833349609375,
      "completions/mean_terminated_length": 732.4203979492188,
      "completions/min_length": 414.0,
      "completions/min_terminated_length": 414.0,
      "entropy": 0.6987628817558289,
      "epoch": 0.0067506750675067504,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.09375,
      "learning_rate": 8.944844124700239e-07,
      "loss": -0.0326,
      "num_tokens": 5020732.0,
      "reward": 0.2718124330043793,
      "reward_std": 0.27098544836044314,
      "rewards/ADEnReward/mean": 0.030245957523584367,
      "rewards/ADEnReward/std": 0.07657596468925476,
      "rewards/CriticV2Reward/mean": 0.15239980369806289,
      "rewards/CriticV2Reward/std": 0.23993462324142456,
      "rewards/StrictFormatReward/mean": 0.8916666626930236,
      "rewards/StrictFormatReward/std": 0.44593390822410583,
      "sampling/importance_sampling_ratio/max": 2.7716786861419678,
      "sampling/importance_sampling_ratio/mean": 0.37447538375854494,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.8250603675842285,
      "sampling/sampling_logp_difference/mean": 0.028124269843101502,
      "step": 45,
      "step_time": 14.862587895337493
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 824.8,
      "completions/max_terminated_length": 824.8,
      "completions/mean_length": 733.1625122070312,
      "completions/mean_terminated_length": 733.1625122070312,
      "completions/min_length": 667.4,
      "completions/min_terminated_length": 667.4,
      "entropy": 0.6778076231479645,
      "epoch": 0.007500750075007501,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.54296875,
      "learning_rate": 8.82494004796163e-07,
      "loss": -0.0308,
      "num_tokens": 5577763.0,
      "reward": 0.29854954183101656,
      "reward_std": 0.256597700715065,
      "rewards/ADEnReward/mean": 0.03557878658175469,
      "rewards/ADEnReward/std": 0.0968101128935814,
      "rewards/CriticV2Reward/mean": 0.16547072678804398,
      "rewards/CriticV2Reward/std": 0.23895596265792846,
      "rewards/StrictFormatReward/mean": 0.9749999880790711,
      "rewards/StrictFormatReward/std": 0.17320507764816284,
      "sampling/importance_sampling_ratio/max": 2.2660447359085083,
      "sampling/importance_sampling_ratio/mean": 0.37205352783203127,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.8447646379470823,
      "sampling/sampling_logp_difference/mean": 0.027005545049905776,
      "step": 50,
      "step_time": 13.36007147487253
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 831.8,
      "completions/max_terminated_length": 831.8,
      "completions/mean_length": 730.0625244140625,
      "completions/mean_terminated_length": 730.0625244140625,
      "completions/min_length": 428.4,
      "completions/min_terminated_length": 428.4,
      "entropy": 0.6709417462348938,
      "epoch": 0.00825082508250825,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.734375,
      "learning_rate": 8.705035971223021e-07,
      "loss": -0.0405,
      "num_tokens": 6134242.0,
      "reward": 0.3022352814674377,
      "reward_std": 0.28455966114997866,
      "rewards/ADEnReward/mean": 0.02506868541240692,
      "rewards/ADEnReward/std": 0.0622736057266593,
      "rewards/CriticV2Reward/mean": 0.18383326530456542,
      "rewards/CriticV2Reward/std": 0.2717557460069656,
      "rewards/StrictFormatReward/mean": 0.9333333253860474,
      "rewards/StrictFormatReward/std": 0.3111684799194336,
      "sampling/importance_sampling_ratio/max": 2.504676914215088,
      "sampling/importance_sampling_ratio/mean": 0.35438522696495056,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.3409300088882445,
      "sampling/sampling_logp_difference/mean": 0.028006478771567346,
      "step": 55,
      "step_time": 13.314154567103833
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1091.6,
      "completions/max_terminated_length": 1091.6,
      "completions/mean_length": 740.2875122070312,
      "completions/mean_terminated_length": 740.2875122070312,
      "completions/min_length": 645.2,
      "completions/min_terminated_length": 645.2,
      "entropy": 0.6656760215759278,
      "epoch": 0.009000900090009001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.43359375,
      "learning_rate": 8.585131894484412e-07,
      "loss": 0.0216,
      "num_tokens": 6692823.0,
      "reward": 0.2957646012306213,
      "reward_std": 0.2834887892007828,
      "rewards/ADEnReward/mean": 0.023848682641983032,
      "rewards/ADEnReward/std": 0.06435841396450996,
      "rewards/CriticV2Reward/mean": 0.1802492320537567,
      "rewards/CriticV2Reward/std": 0.26042196750640867,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.39194485545158386,
      "sampling/importance_sampling_ratio/max": 1.8143912553787231,
      "sampling/importance_sampling_ratio/mean": 0.30097441375255585,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.154077529907227,
      "sampling/sampling_logp_difference/mean": 0.027227402478456498,
      "step": 60,
      "step_time": 15.13567282371223
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 876.8,
      "completions/max_terminated_length": 876.8,
      "completions/mean_length": 734.7583618164062,
      "completions/mean_terminated_length": 734.7583618164062,
      "completions/min_length": 662.8,
      "completions/min_terminated_length": 662.8,
      "entropy": 0.6781624317169189,
      "epoch": 0.00975097509750975,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.271484375,
      "learning_rate": 8.465227817745802e-07,
      "loss": -0.0528,
      "num_tokens": 7250205.0,
      "reward": 0.2756860703229904,
      "reward_std": 0.23353091776371002,
      "rewards/ADEnReward/mean": 0.03827684186398983,
      "rewards/ADEnReward/std": 0.07406068071722985,
      "rewards/CriticV2Reward/mean": 0.1440758764743805,
      "rewards/CriticV2Reward/std": 0.21903067827224731,
      "rewards/StrictFormatReward/mean": 0.9333333253860474,
      "rewards/StrictFormatReward/std": 0.2966887950897217,
      "sampling/importance_sampling_ratio/max": 2.4248700618743895,
      "sampling/importance_sampling_ratio/mean": 0.34985809326171874,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.2400750637054445,
      "sampling/sampling_logp_difference/mean": 0.027471666783094408,
      "step": 65,
      "step_time": 13.655441191699357
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1133.4,
      "completions/max_terminated_length": 985.8,
      "completions/mean_length": 740.595849609375,
      "completions/mean_terminated_length": 737.2568237304688,
      "completions/min_length": 664.0,
      "completions/min_terminated_length": 664.0,
      "entropy": 0.6988117396831512,
      "epoch": 0.010501050105010502,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6484375,
      "learning_rate": 8.345323741007194e-07,
      "loss": -0.059,
      "num_tokens": 7808828.0,
      "reward": 0.22612838745117186,
      "reward_std": 0.19224123358726503,
      "rewards/ADEnReward/mean": 0.020160195766948164,
      "rewards/ADEnReward/std": 0.04457876272499561,
      "rewards/CriticV2Reward/mean": 0.11180151477456093,
      "rewards/CriticV2Reward/std": 0.18313901424407958,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.294095641374588,
      "sampling/importance_sampling_ratio/max": 2.2685441970825195,
      "sampling/importance_sampling_ratio/mean": 0.3226285606622696,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.104148912429809,
      "sampling/sampling_logp_difference/mean": 0.027278991416096687,
      "step": 70,
      "step_time": 15.464104375429452
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 937.2,
      "completions/max_terminated_length": 937.2,
      "completions/mean_length": 735.0416870117188,
      "completions/mean_terminated_length": 735.0416870117188,
      "completions/min_length": 660.0,
      "completions/min_terminated_length": 660.0,
      "entropy": 0.6734846651554107,
      "epoch": 0.011251125112511251,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8671875,
      "learning_rate": 8.225419664268585e-07,
      "loss": 0.0331,
      "num_tokens": 8366182.0,
      "reward": 0.2327167123556137,
      "reward_std": 0.20908381342887877,
      "rewards/ADEnReward/mean": 0.02178483735769987,
      "rewards/ADEnReward/std": 0.0545375794172287,
      "rewards/CriticV2Reward/mean": 0.11926519349217415,
      "rewards/CriticV2Reward/std": 0.18717683404684066,
      "rewards/StrictFormatReward/mean": 0.9166666746139527,
      "rewards/StrictFormatReward/std": 0.34277149438858034,
      "sampling/importance_sampling_ratio/max": 2.570010709762573,
      "sampling/importance_sampling_ratio/mean": 0.41035584211349485,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.5141350746154787,
      "sampling/sampling_logp_difference/mean": 0.027138520777225495,
      "step": 75,
      "step_time": 14.170246110949666
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 825.2,
      "completions/max_terminated_length": 825.2,
      "completions/mean_length": 731.9916870117188,
      "completions/mean_terminated_length": 731.9916870117188,
      "completions/min_length": 659.0,
      "completions/min_terminated_length": 659.0,
      "entropy": 0.6663954973220825,
      "epoch": 0.012001200120012,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.2734375,
      "learning_rate": 8.105515587529975e-07,
      "loss": 0.0095,
      "num_tokens": 8922980.0,
      "reward": 0.29745977818965913,
      "reward_std": 0.28679852187633514,
      "rewards/ADEnReward/mean": 0.03675286723300815,
      "rewards/ADEnReward/std": 0.10012084320187568,
      "rewards/CriticV2Reward/mean": 0.16487355828285216,
      "rewards/CriticV2Reward/std": 0.2441360980272293,
      "rewards/StrictFormatReward/mean": 0.9583333253860473,
      "rewards/StrictFormatReward/std": 0.21331926584243774,
      "sampling/importance_sampling_ratio/max": 2.6717817783355713,
      "sampling/importance_sampling_ratio/mean": 0.40106405019760133,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.966424179077149,
      "sampling/sampling_logp_difference/mean": 0.0272501390427351,
      "step": 80,
      "step_time": 13.397188885137439
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 824.0,
      "completions/max_terminated_length": 824.0,
      "completions/mean_length": 728.7416870117188,
      "completions/mean_terminated_length": 728.7416870117188,
      "completions/min_length": 578.0,
      "completions/min_terminated_length": 578.0,
      "entropy": 0.6762633740901947,
      "epoch": 0.012751275127512752,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.83203125,
      "learning_rate": 7.985611510791367e-07,
      "loss": -0.0382,
      "num_tokens": 9478630.0,
      "reward": 0.29869637489318845,
      "reward_std": 0.2758790373802185,
      "rewards/ADEnReward/mean": 0.021353329718112945,
      "rewards/ADEnReward/std": 0.052459438890218736,
      "rewards/CriticV2Reward/mean": 0.17984303086996078,
      "rewards/CriticV2Reward/std": 0.2615115612745285,
      "rewards/StrictFormatReward/mean": 0.975,
      "rewards/StrictFormatReward/std": 0.13851140141487123,
      "sampling/importance_sampling_ratio/max": 2.4585710525512696,
      "sampling/importance_sampling_ratio/mean": 0.36855998635292053,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.735863780975342,
      "sampling/sampling_logp_difference/mean": 0.027072810009121893,
      "step": 85,
      "step_time": 13.575490239448845
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 945.0,
      "completions/max_terminated_length": 945.0,
      "completions/mean_length": 728.6083618164063,
      "completions/mean_terminated_length": 728.6083618164063,
      "completions/min_length": 535.6,
      "completions/min_terminated_length": 535.6,
      "entropy": 0.6834039151668548,
      "epoch": 0.013501350135013501,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.421875,
      "learning_rate": 7.865707434052757e-07,
      "loss": -0.0357,
      "num_tokens": 10034216.0,
      "reward": 0.25413125455379487,
      "reward_std": 0.23914098739624023,
      "rewards/ADEnReward/mean": 0.04056290723383427,
      "rewards/ADEnReward/std": 0.09261687994003295,
      "rewards/CriticV2Reward/mean": 0.12273500114679337,
      "rewards/CriticV2Reward/std": 0.2090004175901413,
      "rewards/StrictFormatReward/mean": 0.9083333373069763,
      "rewards/StrictFormatReward/std": 0.31918213367462156,
      "sampling/importance_sampling_ratio/max": 2.5320541858673096,
      "sampling/importance_sampling_ratio/mean": 0.4195625066757202,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.6738919258117675,
      "sampling/sampling_logp_difference/mean": 0.02621118016541004,
      "step": 90,
      "step_time": 14.163981208670885
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1055.0,
      "completions/max_terminated_length": 1055.0,
      "completions/mean_length": 734.9208618164063,
      "completions/mean_terminated_length": 734.9208618164063,
      "completions/min_length": 625.0,
      "completions/min_terminated_length": 625.0,
      "entropy": 0.6778877079486847,
      "epoch": 0.014251425142514252,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.54296875,
      "learning_rate": 7.745803357314147e-07,
      "loss": 0.0453,
      "num_tokens": 10591749.0,
      "reward": 0.2550377070903778,
      "reward_std": 0.24310122430324554,
      "rewards/ADEnReward/mean": 0.011902040056884288,
      "rewards/ADEnReward/std": 0.032011125981807706,
      "rewards/CriticV2Reward/mean": 0.14980231374502181,
      "rewards/CriticV2Reward/std": 0.2343314677476883,
      "rewards/StrictFormatReward/mean": 0.9333333373069763,
      "rewards/StrictFormatReward/std": 0.3577991783618927,
      "sampling/importance_sampling_ratio/max": 2.2254454612731935,
      "sampling/importance_sampling_ratio/mean": 0.33642452359199526,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.6567583084106445,
      "sampling/sampling_logp_difference/mean": 0.02788180038332939,
      "step": 95,
      "step_time": 15.019571543950587
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 805.8,
      "completions/max_terminated_length": 805.8,
      "completions/mean_length": 724.3875244140625,
      "completions/mean_terminated_length": 724.3875244140625,
      "completions/min_length": 638.0,
      "completions/min_terminated_length": 638.0,
      "entropy": 0.6795200049877167,
      "epoch": 0.015001500150015001,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.458984375,
      "learning_rate": 7.62589928057554e-07,
      "loss": -0.0563,
      "num_tokens": 11145906.0,
      "reward": 0.27770817279815674,
      "reward_std": 0.2380681112408638,
      "rewards/ADEnReward/mean": 0.048516617715358736,
      "rewards/ADEnReward/std": 0.1014238491654396,
      "rewards/CriticV2Reward/mean": 0.13669154196977615,
      "rewards/CriticV2Reward/std": 0.20003681480884553,
      "rewards/StrictFormatReward/mean": 0.924999988079071,
      "rewards/StrictFormatReward/std": 0.36570553183555604,
      "sampling/importance_sampling_ratio/max": 2.1466657400131224,
      "sampling/importance_sampling_ratio/mean": 0.3968630850315094,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.029999828338623,
      "sampling/sampling_logp_difference/mean": 0.026145283505320548,
      "step": 100,
      "step_time": 13.395550060458481
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 816.4,
      "completions/max_terminated_length": 816.4,
      "completions/mean_length": 735.120849609375,
      "completions/mean_terminated_length": 735.120849609375,
      "completions/min_length": 674.0,
      "completions/min_terminated_length": 674.0,
      "entropy": 0.6767585337162018,
      "epoch": 0.01575157515751575,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5078125,
      "learning_rate": 7.50599520383693e-07,
      "loss": -0.0046,
      "num_tokens": 11703711.0,
      "reward": 0.22518396079540254,
      "reward_std": 0.21424060463905334,
      "rewards/ADEnReward/mean": 0.02892113085836172,
      "rewards/ADEnReward/std": 0.07816565409302711,
      "rewards/CriticV2Reward/mean": 0.10376282632350922,
      "rewards/CriticV2Reward/std": 0.1866258502006531,
      "rewards/StrictFormatReward/mean": 0.924999988079071,
      "rewards/StrictFormatReward/std": 0.36890350580215453,
      "sampling/importance_sampling_ratio/max": 2.333189105987549,
      "sampling/importance_sampling_ratio/mean": 0.3842966675758362,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.3249330043792726,
      "sampling/sampling_logp_difference/mean": 0.027384335547685622,
      "step": 105,
      "step_time": 13.222316846810282
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 812.2,
      "completions/max_terminated_length": 812.2,
      "completions/mean_length": 727.433349609375,
      "completions/mean_terminated_length": 727.433349609375,
      "completions/min_length": 646.4,
      "completions/min_terminated_length": 646.4,
      "entropy": 0.6769951701164245,
      "epoch": 0.0165016501650165,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.53515625,
      "learning_rate": 7.386091127098321e-07,
      "loss": -0.0397,
      "num_tokens": 12258999.0,
      "reward": 0.23659175038337707,
      "reward_std": 0.2429301619529724,
      "rewards/ADEnReward/mean": 0.02100809020921588,
      "rewards/ADEnReward/std": 0.06905662044882774,
      "rewards/CriticV2Reward/mean": 0.12225031852722168,
      "rewards/CriticV2Reward/std": 0.21501878201961516,
      "rewards/StrictFormatReward/mean": 0.9333333373069763,
      "rewards/StrictFormatReward/std": 0.3171369910240173,
      "sampling/importance_sampling_ratio/max": 2.3933506488800047,
      "sampling/importance_sampling_ratio/mean": 0.34748960137367246,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.1887176513671873,
      "sampling/sampling_logp_difference/mean": 0.02707708366215229,
      "step": 110,
      "step_time": 13.317681908141822
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1086.4,
      "completions/max_terminated_length": 945.4,
      "completions/mean_length": 739.9375244140625,
      "completions/mean_terminated_length": 736.5984985351563,
      "completions/min_length": 557.4,
      "completions/min_terminated_length": 557.4,
      "entropy": 0.6870240211486817,
      "epoch": 0.017251725172517253,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.330078125,
      "learning_rate": 7.266187050359712e-07,
      "loss": 0.0218,
      "num_tokens": 12817736.0,
      "reward": 0.25257408022880556,
      "reward_std": 0.2376851588487625,
      "rewards/ADEnReward/mean": 0.027911533229053022,
      "rewards/ADEnReward/std": 0.07045800052583218,
      "rewards/CriticV2Reward/mean": 0.13216253668069838,
      "rewards/CriticV2Reward/std": 0.21919724345207214,
      "rewards/StrictFormatReward/mean": 0.925,
      "rewards/StrictFormatReward/std": 0.37487201690673827,
      "sampling/importance_sampling_ratio/max": 2.0126373291015627,
      "sampling/importance_sampling_ratio/mean": 0.35888453125953673,
      "sampling/importance_sampling_ratio/min": 5.0439726328477265e-05,
      "sampling/sampling_logp_difference/max": 3.8658429384231567,
      "sampling/sampling_logp_difference/mean": 0.02792995758354664,
      "step": 115,
      "step_time": 14.98624010598287
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 813.2,
      "completions/max_terminated_length": 813.2,
      "completions/mean_length": 727.6916870117187,
      "completions/mean_terminated_length": 727.6916870117187,
      "completions/min_length": 538.4,
      "completions/min_terminated_length": 538.4,
      "entropy": 0.6622766137123108,
      "epoch": 0.018001800180018002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4453125,
      "learning_rate": 7.146282973621102e-07,
      "loss": -0.0632,
      "num_tokens": 13373374.0,
      "reward": 0.2659685522317886,
      "reward_std": 0.2437886893749237,
      "rewards/ADEnReward/mean": 0.023663881793618204,
      "rewards/ADEnReward/std": 0.05939733125269413,
      "rewards/CriticV2Reward/mean": 0.14813800007104874,
      "rewards/CriticV2Reward/std": 0.21836518943309785,
      "rewards/StrictFormatReward/mean": 0.9416666746139526,
      "rewards/StrictFormatReward/std": 0.30006415247917173,
      "sampling/importance_sampling_ratio/max": 2.6608290672302246,
      "sampling/importance_sampling_ratio/mean": 0.40939287543296815,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.3562741041183473,
      "sampling/sampling_logp_difference/mean": 0.027171416208148003,
      "step": 120,
      "step_time": 13.383991296868771
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00833333358168602,
      "completions/max_length": 1089.8,
      "completions/max_terminated_length": 798.4,
      "completions/mean_length": 738.9291870117188,
      "completions/mean_terminated_length": 732.2929931640625,
      "completions/min_length": 654.0,
      "completions/min_terminated_length": 654.0,
      "entropy": 0.6830495119094848,
      "epoch": 0.01875187518751875,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.478515625,
      "learning_rate": 7.026378896882493e-07,
      "loss": 0.0473,
      "num_tokens": 13931613.0,
      "reward": 0.26927419006824493,
      "reward_std": 0.24791643023490906,
      "rewards/ADEnReward/mean": 0.029334013583138585,
      "rewards/ADEnReward/std": 0.06973049007356166,
      "rewards/CriticV2Reward/mean": 0.1432734966278076,
      "rewards/CriticV2Reward/std": 0.2263641268014908,
      "rewards/StrictFormatReward/mean": 0.9666666507720947,
      "rewards/StrictFormatReward/std": 0.2309401035308838,
      "sampling/importance_sampling_ratio/max": 2.6443384647369386,
      "sampling/importance_sampling_ratio/mean": 0.44446309208869933,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.7472902774810795,
      "sampling/sampling_logp_difference/mean": 0.027243128046393394,
      "step": 125,
      "step_time": 15.178255619574339
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 964.0,
      "completions/max_terminated_length": 816.8,
      "completions/mean_length": 735.46669921875,
      "completions/mean_terminated_length": 732.1085327148437,
      "completions/min_length": 608.0,
      "completions/min_terminated_length": 608.0,
      "entropy": 0.6759671211242676,
      "epoch": 0.0195019501950195,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.48046875,
      "learning_rate": 6.906474820143885e-07,
      "loss": 0.025,
      "num_tokens": 14489661.0,
      "reward": 0.31296299397945404,
      "reward_std": 0.2914800405502319,
      "rewards/ADEnReward/mean": 0.034117148048244415,
      "rewards/ADEnReward/std": 0.08479407420381904,
      "rewards/CriticV2Reward/mean": 0.18801250755786897,
      "rewards/CriticV2Reward/std": 0.26688812375068666,
      "rewards/StrictFormatReward/mean": 0.9083333373069763,
      "rewards/StrictFormatReward/std": 0.3529752492904663,
      "sampling/importance_sampling_ratio/max": 2.6238924980163576,
      "sampling/importance_sampling_ratio/mean": 0.36424337029457093,
      "sampling/importance_sampling_ratio/min": 9.081272291950881e-05,
      "sampling/sampling_logp_difference/max": 2.2420372009277343,
      "sampling/sampling_logp_difference/mean": 0.027383123710751533,
      "step": 130,
      "step_time": 14.45475140977651
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 817.4,
      "completions/max_terminated_length": 817.4,
      "completions/mean_length": 730.6375244140625,
      "completions/mean_terminated_length": 730.6375244140625,
      "completions/min_length": 547.2,
      "completions/min_terminated_length": 547.2,
      "entropy": 0.6457026600837708,
      "epoch": 0.02025202520252025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.7265625,
      "learning_rate": 6.786570743405276e-07,
      "loss": -0.0069,
      "num_tokens": 15046086.0,
      "reward": 0.30475842356681826,
      "reward_std": 0.29704451858997344,
      "rewards/ADEnReward/mean": 0.029951873375102876,
      "rewards/ADEnReward/std": 0.0687396578490734,
      "rewards/CriticV2Reward/mean": 0.18313985466957092,
      "rewards/CriticV2Reward/std": 0.2745169162750244,
      "rewards/StrictFormatReward/mean": 0.9166666746139527,
      "rewards/StrictFormatReward/std": 0.30210930705070493,
      "sampling/importance_sampling_ratio/max": 2.5814715385437013,
      "sampling/importance_sampling_ratio/mean": 0.4488029360771179,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.662810564041138,
      "sampling/sampling_logp_difference/mean": 0.026542375236749648,
      "step": 135,
      "step_time": 13.331754672527314
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 802.8,
      "completions/max_terminated_length": 802.8,
      "completions/mean_length": 734.3500122070312,
      "completions/mean_terminated_length": 734.3500122070312,
      "completions/min_length": 667.2,
      "completions/min_terminated_length": 667.2,
      "entropy": 0.6539111316204071,
      "epoch": 0.021002100210021003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2578125,
      "learning_rate": 6.666666666666666e-07,
      "loss": 0.0054,
      "num_tokens": 15603610.0,
      "reward": 0.32718279361724856,
      "reward_std": 0.2756353974342346,
      "rewards/ADEnReward/mean": 0.035100750438869,
      "rewards/ADEnReward/std": 0.0759646624326706,
      "rewards/CriticV2Reward/mean": 0.19708201587200164,
      "rewards/CriticV2Reward/std": 0.2542935281991959,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.27702280282974245,
      "sampling/importance_sampling_ratio/max": 1.9317828178405763,
      "sampling/importance_sampling_ratio/mean": 0.2619578391313553,
      "sampling/importance_sampling_ratio/min": 0.0020720528438687323,
      "sampling/sampling_logp_difference/max": 3.8336150884628295,
      "sampling/sampling_logp_difference/mean": 0.02702697478234768,
      "step": 140,
      "step_time": 13.271004169993102
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 864.4,
      "completions/max_terminated_length": 864.4,
      "completions/mean_length": 728.5625,
      "completions/mean_terminated_length": 728.5625,
      "completions/min_length": 560.4,
      "completions/min_terminated_length": 560.4,
      "entropy": 0.6553530871868134,
      "epoch": 0.021752175217521753,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.2890625,
      "learning_rate": 6.546762589928057e-07,
      "loss": -0.0144,
      "num_tokens": 16159489.0,
      "reward": 0.3294836848974228,
      "reward_std": 0.3109097331762314,
      "rewards/ADEnReward/mean": 0.03947688639163971,
      "rewards/ADEnReward/std": 0.08423196449875832,
      "rewards/CriticV2Reward/mean": 0.19584011435508727,
      "rewards/CriticV2Reward/std": 0.2782162755727768,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.25343345403671264,
      "sampling/importance_sampling_ratio/max": 2.2559653520584106,
      "sampling/importance_sampling_ratio/mean": 0.39474677443504336,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.220552110671997,
      "sampling/sampling_logp_difference/mean": 0.026592973992228507,
      "step": 145,
      "step_time": 13.792286728322505
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 816.2,
      "completions/max_terminated_length": 816.2,
      "completions/mean_length": 727.458349609375,
      "completions/mean_terminated_length": 727.458349609375,
      "completions/min_length": 528.8,
      "completions/min_terminated_length": 528.8,
      "entropy": 0.6875123620033264,
      "epoch": 0.022502250225022502,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.82421875,
      "learning_rate": 6.426858513189448e-07,
      "loss": 0.0156,
      "num_tokens": 16715023.0,
      "reward": 0.2460826128721237,
      "reward_std": 0.2182670548558235,
      "rewards/ADEnReward/mean": 0.021830105502158404,
      "rewards/ADEnReward/std": 0.05803550221025944,
      "rewards/CriticV2Reward/mean": 0.1309191606938839,
      "rewards/CriticV2Reward/std": 0.20253435969352723,
      "rewards/StrictFormatReward/mean": 0.9333333253860474,
      "rewards/StrictFormatReward/std": 0.30797049403190613,
      "sampling/importance_sampling_ratio/max": 2.603206253051758,
      "sampling/importance_sampling_ratio/mean": 0.3743092954158783,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.419530653953552,
      "sampling/sampling_logp_difference/mean": 0.027796153724193574,
      "step": 150,
      "step_time": 13.406930388044566
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1140.0,
      "completions/max_terminated_length": 995.0,
      "completions/mean_length": 746.908349609375,
      "completions/mean_terminated_length": 743.5602905273438,
      "completions/min_length": 667.6,
      "completions/min_terminated_length": 667.6,
      "entropy": 0.6762481391429901,
      "epoch": 0.02325232523252325,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.4609375,
      "learning_rate": 6.306954436450838e-07,
      "loss": -0.027,
      "num_tokens": 17275209.0,
      "reward": 0.319897449016571,
      "reward_std": 0.2809853881597519,
      "rewards/ADEnReward/mean": 0.046538510359823704,
      "rewards/ADEnReward/std": 0.09964709505438804,
      "rewards/CriticV2Reward/mean": 0.1800255909562111,
      "rewards/CriticV2Reward/std": 0.2593708336353302,
      "rewards/StrictFormatReward/mean": 0.9333333253860474,
      "rewards/StrictFormatReward/std": 0.3111684799194336,
      "sampling/importance_sampling_ratio/max": 1.9672291159629822,
      "sampling/importance_sampling_ratio/mean": 0.37010484337806704,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.342987585067749,
      "sampling/sampling_logp_difference/mean": 0.027148356661200523,
      "step": 155,
      "step_time": 15.492892314493655
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 847.4,
      "completions/max_terminated_length": 847.4,
      "completions/mean_length": 730.895849609375,
      "completions/mean_terminated_length": 730.895849609375,
      "completions/min_length": 670.2,
      "completions/min_terminated_length": 670.2,
      "entropy": 0.6781861484050751,
      "epoch": 0.024002400240024,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8359375,
      "learning_rate": 6.187050359712231e-07,
      "loss": -0.0039,
      "num_tokens": 17831280.0,
      "reward": 0.2673605889081955,
      "reward_std": 0.2609496355056763,
      "rewards/ADEnReward/mean": 0.022430585324764253,
      "rewards/ADEnReward/std": 0.05508136935532093,
      "rewards/CriticV2Reward/mean": 0.14826332926750183,
      "rewards/CriticV2Reward/std": 0.23569288849830627,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.19624642729759217,
      "sampling/importance_sampling_ratio/max": 2.6008774757385256,
      "sampling/importance_sampling_ratio/mean": 0.34396409392356875,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.0486708641052247,
      "sampling/sampling_logp_difference/mean": 0.0269806656986475,
      "step": 160,
      "step_time": 13.56647696653381
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 803.2,
      "completions/max_terminated_length": 803.2,
      "completions/mean_length": 723.6166748046875,
      "completions/mean_terminated_length": 723.6166748046875,
      "completions/min_length": 542.0,
      "completions/min_terminated_length": 542.0,
      "entropy": 0.6606142938137054,
      "epoch": 0.024752475247524754,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9453125,
      "learning_rate": 6.067146282973621e-07,
      "loss": -0.0583,
      "num_tokens": 18385348.0,
      "reward": 0.2865483522415161,
      "reward_std": 0.2430926740169525,
      "rewards/ADEnReward/mean": 0.038159405998885634,
      "rewards/ADEnReward/std": 0.09702088348567486,
      "rewards/CriticV2Reward/mean": 0.15088892579078675,
      "rewards/CriticV2Reward/std": 0.22967694103717803,
      "rewards/StrictFormatReward/mean": 0.9749999880790711,
      "rewards/StrictFormatReward/std": 0.17320507764816284,
      "sampling/importance_sampling_ratio/max": 2.3871933221817017,
      "sampling/importance_sampling_ratio/mean": 0.3935186803340912,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.7643911838531494,
      "sampling/sampling_logp_difference/mean": 0.02641451619565487,
      "step": 165,
      "step_time": 13.218168735411018
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 952.4,
      "completions/max_terminated_length": 808.0,
      "completions/mean_length": 730.1541870117187,
      "completions/mean_terminated_length": 726.7688110351562,
      "completions/min_length": 427.0,
      "completions/min_terminated_length": 427.0,
      "entropy": 0.6817957520484924,
      "epoch": 0.025502550255025503,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.466796875,
      "learning_rate": 5.947242206235011e-07,
      "loss": -0.11,
      "num_tokens": 18941433.0,
      "reward": 0.2676409512758255,
      "reward_std": 0.24391051530838012,
      "rewards/ADEnReward/mean": 0.03495770245790482,
      "rewards/ADEnReward/std": 0.08008336126804352,
      "rewards/CriticV2Reward/mean": 0.14184990525245667,
      "rewards/CriticV2Reward/std": 0.2124081790447235,
      "rewards/StrictFormatReward/mean": 0.9083333373069763,
      "rewards/StrictFormatReward/std": 0.4005065083503723,
      "sampling/importance_sampling_ratio/max": 2.514456629753113,
      "sampling/importance_sampling_ratio/mean": 0.3926523685455322,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.343113994598388,
      "sampling/sampling_logp_difference/mean": 0.027290407940745352,
      "step": 170,
      "step_time": 14.132117554545403
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 952.8,
      "completions/max_terminated_length": 952.8,
      "completions/mean_length": 734.9291748046875,
      "completions/mean_terminated_length": 734.9291748046875,
      "completions/min_length": 572.0,
      "completions/min_terminated_length": 572.0,
      "entropy": 0.6970304548740387,
      "epoch": 0.026252625262526252,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.69921875,
      "learning_rate": 5.827338129496403e-07,
      "loss": 0.0331,
      "num_tokens": 19498808.0,
      "reward": 0.27273200154304506,
      "reward_std": 0.2695725470781326,
      "rewards/ADEnReward/mean": 0.01761272009462118,
      "rewards/ADEnReward/std": 0.06024995818734169,
      "rewards/CriticV2Reward/mean": 0.1609526053071022,
      "rewards/CriticV2Reward/std": 0.25695641338825226,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.25343345403671264,
      "sampling/importance_sampling_ratio/max": 2.529092001914978,
      "sampling/importance_sampling_ratio/mean": 0.39281960129737853,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.5247619390487674,
      "sampling/sampling_logp_difference/mean": 0.026818586885929106,
      "step": 175,
      "step_time": 14.21546617038548
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 831.6,
      "completions/max_terminated_length": 831.6,
      "completions/mean_length": 732.2125122070313,
      "completions/mean_terminated_length": 732.2125122070313,
      "completions/min_length": 675.6,
      "completions/min_terminated_length": 675.6,
      "entropy": 0.6531910359859466,
      "epoch": 0.027002700270027002,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.484375,
      "learning_rate": 5.707434052757793e-07,
      "loss": -0.0202,
      "num_tokens": 20055611.0,
      "reward": 0.314579039812088,
      "reward_std": 0.2669638603925705,
      "rewards/ADEnReward/mean": 0.050490521267056464,
      "rewards/ADEnReward/std": 0.09671776592731476,
      "rewards/CriticV2Reward/mean": 0.1657551869750023,
      "rewards/CriticV2Reward/std": 0.2505346149206161,
      "rewards/StrictFormatReward/mean": 0.9833333373069764,
      "rewards/StrictFormatReward/std": 0.08077637553215027,
      "sampling/importance_sampling_ratio/max": 2.2050620555877685,
      "sampling/importance_sampling_ratio/mean": 0.3497112452983856,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.723379349708557,
      "sampling/sampling_logp_difference/mean": 0.026673595979809762,
      "step": 180,
      "step_time": 13.579954623430968
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00833333358168602,
      "completions/max_length": 1110.2,
      "completions/max_terminated_length": 821.6,
      "completions/mean_length": 738.7416870117188,
      "completions/mean_terminated_length": 731.9643676757812,
      "completions/min_length": 656.6,
      "completions/min_terminated_length": 656.6,
      "entropy": 0.6632063090801239,
      "epoch": 0.02775277527752775,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.58203125,
      "learning_rate": 5.587529976019184e-07,
      "loss": -0.0011,
      "num_tokens": 20614061.0,
      "reward": 0.2985087275505066,
      "reward_std": 0.2817197978496552,
      "rewards/ADEnReward/mean": 0.04548268616199493,
      "rewards/ADEnReward/std": 0.10240544080734253,
      "rewards/CriticV2Reward/mean": 0.16135935932397844,
      "rewards/CriticV2Reward/std": 0.24745513498783112,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.29294281005859374,
      "sampling/importance_sampling_ratio/max": 2.2299548625946044,
      "sampling/importance_sampling_ratio/mean": 0.36481807231903074,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.58858847618103,
      "sampling/sampling_logp_difference/mean": 0.027159354835748672,
      "step": 185,
      "step_time": 15.367878540046513
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 817.8,
      "completions/max_terminated_length": 817.8,
      "completions/mean_length": 732.2916870117188,
      "completions/mean_terminated_length": 732.2916870117188,
      "completions/min_length": 536.8,
      "completions/min_terminated_length": 536.8,
      "entropy": 0.6737968623638153,
      "epoch": 0.028502850285028504,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.67578125,
      "learning_rate": 5.467625899280576e-07,
      "loss": -0.017,
      "num_tokens": 21170995.0,
      "reward": 0.2543010950088501,
      "reward_std": 0.237994921207428,
      "rewards/ADEnReward/mean": 0.03096686936914921,
      "rewards/ADEnReward/std": 0.07621146515011787,
      "rewards/CriticV2Reward/mean": 0.13000087440013885,
      "rewards/CriticV2Reward/std": 0.2169148951768875,
      "rewards/StrictFormatReward/mean": 0.9333333253860474,
      "rewards/StrictFormatReward/std": 0.35183066725730894,
      "sampling/importance_sampling_ratio/max": 2.219161319732666,
      "sampling/importance_sampling_ratio/mean": 0.3931430697441101,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.842717552185059,
      "sampling/sampling_logp_difference/mean": 0.0267446331679821,
      "step": 190,
      "step_time": 13.365200480632485
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 806.2,
      "completions/max_terminated_length": 806.2,
      "completions/mean_length": 726.0291870117187,
      "completions/mean_terminated_length": 726.0291870117187,
      "completions/min_length": 591.6,
      "completions/min_terminated_length": 591.6,
      "entropy": 0.6693301379680634,
      "epoch": 0.029252925292529253,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.703125,
      "learning_rate": 5.347721822541966e-07,
      "loss": -0.0212,
      "num_tokens": 21725930.0,
      "reward": 0.2516060918569565,
      "reward_std": 0.22183270156383514,
      "rewards/ADEnReward/mean": 0.030396451149135827,
      "rewards/ADEnReward/std": 0.07066891826689244,
      "rewards/CriticV2Reward/mean": 0.128709626942873,
      "rewards/CriticV2Reward/std": 0.20975570380687714,
      "rewards/StrictFormatReward/mean": 0.925000011920929,
      "rewards/StrictFormatReward/std": 0.380840528011322,
      "sampling/importance_sampling_ratio/max": 2.3974624156951903,
      "sampling/importance_sampling_ratio/mean": 0.35138731002807616,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.941667985916138,
      "sampling/sampling_logp_difference/mean": 0.026314648613333703,
      "step": 195,
      "step_time": 13.368891882430763
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 954.2,
      "completions/max_terminated_length": 954.2,
      "completions/mean_length": 736.1416748046875,
      "completions/mean_terminated_length": 736.1416748046875,
      "completions/min_length": 668.0,
      "completions/min_terminated_length": 668.0,
      "entropy": 0.6977671384811401,
      "epoch": 0.030003000300030003,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.60546875,
      "learning_rate": 5.227817745803357e-07,
      "loss": 0.0251,
      "num_tokens": 22283580.0,
      "reward": 0.25325133800506594,
      "reward_std": 0.23282511830329894,
      "rewards/ADEnReward/mean": 0.013752670167014003,
      "rewards/ADEnReward/std": 0.03788870992138982,
      "rewards/CriticV2Reward/mean": 0.14449864774942398,
      "rewards/CriticV2Reward/std": 0.22673717141151428,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.27702280282974245,
      "sampling/importance_sampling_ratio/max": 1.8331253051757812,
      "sampling/importance_sampling_ratio/mean": 0.32110574245452883,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 6.006078147888184,
      "sampling/sampling_logp_difference/mean": 0.027469008043408395,
      "step": 200,
      "step_time": 14.366346049588174
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 814.6,
      "completions/max_terminated_length": 814.6,
      "completions/mean_length": 729.6416870117188,
      "completions/mean_terminated_length": 729.6416870117188,
      "completions/min_length": 548.8,
      "completions/min_terminated_length": 548.8,
      "entropy": 0.6385031104087829,
      "epoch": 0.030753075307530752,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.83203125,
      "learning_rate": 5.107913669064748e-07,
      "loss": -0.0242,
      "num_tokens": 22839654.0,
      "reward": 0.3597941994667053,
      "reward_std": 0.32995885014534,
      "rewards/ADEnReward/mean": 0.0433755524456501,
      "rewards/ADEnReward/std": 0.0839846022427082,
      "rewards/CriticV2Reward/mean": 0.22225197553634643,
      "rewards/CriticV2Reward/std": 0.2984564691781998,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.3347578287124634,
      "sampling/importance_sampling_ratio/max": 2.4754393100738525,
      "sampling/importance_sampling_ratio/mean": 0.4427867650985718,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.708860540390015,
      "sampling/sampling_logp_difference/mean": 0.02616884931921959,
      "step": 205,
      "step_time": 13.387136839143931
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 921.0,
      "completions/max_terminated_length": 921.0,
      "completions/mean_length": 729.7416870117188,
      "completions/mean_terminated_length": 729.7416870117188,
      "completions/min_length": 545.2,
      "completions/min_terminated_length": 545.2,
      "entropy": 0.6550398588180542,
      "epoch": 0.0315031503150315,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.66796875,
      "learning_rate": 4.988009592326139e-07,
      "loss": 0.0169,
      "num_tokens": 23395880.0,
      "reward": 0.26131516993045806,
      "reward_std": 0.2235265478491783,
      "rewards/ADEnReward/mean": 0.015563993342220783,
      "rewards/ADEnReward/std": 0.04172728545963764,
      "rewards/CriticV2Reward/mean": 0.15325116813182832,
      "rewards/CriticV2Reward/std": 0.22710732668638228,
      "rewards/StrictFormatReward/mean": 0.924999988079071,
      "rewards/StrictFormatReward/std": 0.3657055199146271,
      "sampling/importance_sampling_ratio/max": 2.1074886083602906,
      "sampling/importance_sampling_ratio/mean": 0.317373925447464,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.4951153755187985,
      "sampling/sampling_logp_difference/mean": 0.026737869903445243,
      "step": 210,
      "step_time": 14.018542888667435
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 829.2,
      "completions/max_terminated_length": 829.2,
      "completions/mean_length": 737.1625244140625,
      "completions/mean_terminated_length": 737.1625244140625,
      "completions/min_length": 663.2,
      "completions/min_terminated_length": 663.2,
      "entropy": 0.6886229276657104,
      "epoch": 0.032253225322532254,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.609375,
      "learning_rate": 4.86810551558753e-07,
      "loss": -0.0492,
      "num_tokens": 23954047.0,
      "reward": 0.28127689361572267,
      "reward_std": 0.24524502754211425,
      "rewards/ADEnReward/mean": 0.030380566231906415,
      "rewards/ADEnReward/std": 0.06638182587921619,
      "rewards/CriticV2Reward/mean": 0.15589632242918014,
      "rewards/CriticV2Reward/std": 0.22414239943027497,
      "rewards/StrictFormatReward/mean": 0.950000011920929,
      "rewards/StrictFormatReward/std": 0.2423291265964508,
      "sampling/importance_sampling_ratio/max": 2.2944180488586428,
      "sampling/importance_sampling_ratio/mean": 0.35170907974243165,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.074922227859497,
      "sampling/sampling_logp_difference/mean": 0.02738656513392925,
      "step": 215,
      "step_time": 13.396183334942908
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 918.8,
      "completions/max_terminated_length": 918.8,
      "completions/mean_length": 733.0500122070313,
      "completions/mean_terminated_length": 733.0500122070313,
      "completions/min_length": 664.4,
      "completions/min_terminated_length": 664.4,
      "entropy": 0.6667313754558564,
      "epoch": 0.033003300330033,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.466796875,
      "learning_rate": 4.7482014388489204e-07,
      "loss": 0.0128,
      "num_tokens": 24511323.0,
      "reward": 0.27115337550640106,
      "reward_std": 0.25859386622905733,
      "rewards/ADEnReward/mean": 0.02348469356074929,
      "rewards/ADEnReward/std": 0.06776928193867207,
      "rewards/CriticV2Reward/mean": 0.15766866207122804,
      "rewards/CriticV2Reward/std": 0.2331855744123459,
      "rewards/StrictFormatReward/mean": 0.9,
      "rewards/StrictFormatReward/std": 0.3700480878353119,
      "sampling/importance_sampling_ratio/max": 2.167785573005676,
      "sampling/importance_sampling_ratio/mean": 0.34185417294502257,
      "sampling/importance_sampling_ratio/min": 2.1186770754866303e-05,
      "sampling/sampling_logp_difference/max": 5.053237056732177,
      "sampling/sampling_logp_difference/mean": 0.027166034281253814,
      "step": 220,
      "step_time": 14.092069184035063
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 793.2,
      "completions/max_terminated_length": 793.2,
      "completions/mean_length": 727.0125244140625,
      "completions/mean_terminated_length": 727.0125244140625,
      "completions/min_length": 543.4,
      "completions/min_terminated_length": 543.4,
      "entropy": 0.6576771378517151,
      "epoch": 0.03375337533753375,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.421875,
      "learning_rate": 4.628297362110312e-07,
      "loss": -0.0292,
      "num_tokens": 25066702.0,
      "reward": 0.28278968334197996,
      "reward_std": 0.277101942896843,
      "rewards/ADEnReward/mean": 0.04402967318892479,
      "rewards/ADEnReward/std": 0.10327454507350922,
      "rewards/CriticV2Reward/mean": 0.14209333211183547,
      "rewards/CriticV2Reward/std": 0.22946595251560212,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.19624642729759217,
      "sampling/importance_sampling_ratio/max": 2.48654568195343,
      "sampling/importance_sampling_ratio/mean": 0.37147427201271055,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.846973514556885,
      "sampling/sampling_logp_difference/mean": 0.026473153755068778,
      "step": 225,
      "step_time": 13.257498236745596
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 921.4,
      "completions/max_terminated_length": 921.4,
      "completions/mean_length": 733.683349609375,
      "completions/mean_terminated_length": 733.683349609375,
      "completions/min_length": 661.6,
      "completions/min_terminated_length": 661.6,
      "entropy": 0.6625209391117096,
      "epoch": 0.034503450345034506,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5546875,
      "learning_rate": 4.508393285371702e-07,
      "loss": -0.0321,
      "num_tokens": 25624018.0,
      "reward": 0.310102254152298,
      "reward_std": 0.29136236310005187,
      "rewards/ADEnReward/mean": 0.029331159964203834,
      "rewards/ADEnReward/std": 0.07541338577866555,
      "rewards/CriticV2Reward/mean": 0.18577107340097426,
      "rewards/CriticV2Reward/std": 0.26120834350585936,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.23636061549186707,
      "sampling/importance_sampling_ratio/max": 2.2946805477142336,
      "sampling/importance_sampling_ratio/mean": 0.36944074034690855,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.595243453979492,
      "sampling/sampling_logp_difference/mean": 0.026906453445553778,
      "step": 230,
      "step_time": 14.133570789452643
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1078.8,
      "completions/max_terminated_length": 1055.0,
      "completions/mean_length": 737.0583618164062,
      "completions/mean_terminated_length": 733.7565795898438,
      "completions/min_length": 561.8,
      "completions/min_terminated_length": 561.8,
      "entropy": 0.6722292423248291,
      "epoch": 0.03525352535253525,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.478515625,
      "learning_rate": 4.388489208633093e-07,
      "loss": 0.0154,
      "num_tokens": 26181936.0,
      "reward": 0.300505068898201,
      "reward_std": 0.28410873413085935,
      "rewards/ADEnReward/mean": 0.030564435198903082,
      "rewards/ADEnReward/std": 0.09145040065050125,
      "rewards/CriticV2Reward/mean": 0.178273943066597,
      "rewards/CriticV2Reward/std": 0.26288277208805083,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.34808468222618105,
      "sampling/importance_sampling_ratio/max": 2.5372284412384034,
      "sampling/importance_sampling_ratio/mean": 0.3265813708305359,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.361432075500488,
      "sampling/sampling_logp_difference/mean": 0.02705967016518116,
      "step": 235,
      "step_time": 14.942696164734661
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 950.8,
      "completions/max_terminated_length": 869.6,
      "completions/mean_length": 734.570849609375,
      "completions/mean_terminated_length": 731.2459228515625,
      "completions/min_length": 654.4,
      "completions/min_terminated_length": 654.4,
      "entropy": 0.665759927034378,
      "epoch": 0.036003600360036005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5859375,
      "learning_rate": 4.2685851318944845e-07,
      "loss": -0.0562,
      "num_tokens": 26739273.0,
      "reward": 0.31318188905715943,
      "reward_std": 0.2850573241710663,
      "rewards/ADEnReward/mean": 0.02819870375096798,
      "rewards/ADEnReward/std": 0.0679096095263958,
      "rewards/CriticV2Reward/mean": 0.19248318374156953,
      "rewards/CriticV2Reward/std": 0.27355403900146485,
      "rewards/StrictFormatReward/mean": 0.925000011920929,
      "rewards/StrictFormatReward/std": 0.28503645658493043,
      "sampling/importance_sampling_ratio/max": 2.6505446910858153,
      "sampling/importance_sampling_ratio/mean": 0.3801407516002655,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.5118245124816894,
      "sampling/sampling_logp_difference/mean": 0.02762332819402218,
      "step": 240,
      "step_time": 14.368941375706346
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 948.4,
      "completions/max_terminated_length": 798.4,
      "completions/mean_length": 732.2875122070312,
      "completions/mean_terminated_length": 728.9180908203125,
      "completions/min_length": 651.6,
      "completions/min_terminated_length": 651.6,
      "entropy": 0.6761650085449219,
      "epoch": 0.03675367536753675,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.66015625,
      "learning_rate": 4.148681055155875e-07,
      "loss": 0.0613,
      "num_tokens": 27295806.0,
      "reward": 0.2632507473230362,
      "reward_std": 0.23623428642749786,
      "rewards/ADEnReward/mean": 0.028980655129998922,
      "rewards/ADEnReward/std": 0.06641366388648748,
      "rewards/CriticV2Reward/mean": 0.14260342791676522,
      "rewards/CriticV2Reward/std": 0.2100657716393471,
      "rewards/StrictFormatReward/mean": 0.9166666746139527,
      "rewards/StrictFormatReward/std": 0.3979133665561676,
      "sampling/importance_sampling_ratio/max": 2.3929349422454833,
      "sampling/importance_sampling_ratio/mean": 0.4016247749328613,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.0486182451248167,
      "sampling/sampling_logp_difference/mean": 0.026698941737413405,
      "step": 245,
      "step_time": 14.183489987812937
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 943.8,
      "completions/max_terminated_length": 943.8,
      "completions/mean_length": 733.6750366210938,
      "completions/mean_terminated_length": 733.6750366210938,
      "completions/min_length": 661.6,
      "completions/min_terminated_length": 661.6,
      "entropy": 0.6711142897605896,
      "epoch": 0.0375037503750375,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6328125,
      "learning_rate": 4.028776978417266e-07,
      "loss": -0.0507,
      "num_tokens": 27852816.0,
      "reward": 0.2927826702594757,
      "reward_std": 0.2535986423492432,
      "rewards/ADEnReward/mean": 0.0256393326446414,
      "rewards/ADEnReward/std": 0.059096282720565795,
      "rewards/CriticV2Reward/mean": 0.17380999326705932,
      "rewards/CriticV2Reward/std": 0.2408513218164444,
      "rewards/StrictFormatReward/mean": 0.9333333373069763,
      "rewards/StrictFormatReward/std": 0.3577991783618927,
      "sampling/importance_sampling_ratio/max": 2.528782033920288,
      "sampling/importance_sampling_ratio/mean": 0.38217838406562804,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.319740128517151,
      "sampling/sampling_logp_difference/mean": 0.026942050084471703,
      "step": 250,
      "step_time": 14.233225086890162
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 978.6,
      "completions/max_terminated_length": 978.6,
      "completions/mean_length": 740.983349609375,
      "completions/mean_terminated_length": 740.983349609375,
      "completions/min_length": 660.2,
      "completions/min_terminated_length": 660.2,
      "entropy": 0.6699490308761596,
      "epoch": 0.038253825382538256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.71875,
      "learning_rate": 3.908872901678657e-07,
      "loss": -0.0133,
      "num_tokens": 28411772.0,
      "reward": 0.30716822743415834,
      "reward_std": 0.2654102027416229,
      "rewards/ADEnReward/mean": 0.028111883625388145,
      "rewards/ADEnReward/std": 0.07746810987591743,
      "rewards/CriticV2Reward/mean": 0.18488966822624206,
      "rewards/CriticV2Reward/std": 0.24855301678180694,
      "rewards/StrictFormatReward/mean": 0.9416666746139526,
      "rewards/StrictFormatReward/std": 0.2594019651412964,
      "sampling/importance_sampling_ratio/max": 2.6309361457824707,
      "sampling/importance_sampling_ratio/mean": 0.33823771476745607,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.0065004587173463,
      "sampling/sampling_logp_difference/mean": 0.02728327251970768,
      "step": 255,
      "step_time": 14.422917295061051
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 945.4,
      "completions/max_terminated_length": 945.4,
      "completions/mean_length": 726.4000244140625,
      "completions/mean_terminated_length": 726.4000244140625,
      "completions/min_length": 463.6,
      "completions/min_terminated_length": 463.6,
      "entropy": 0.6654939651489258,
      "epoch": 0.039003900390039,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.5546875,
      "learning_rate": 3.7889688249400476e-07,
      "loss": -0.0193,
      "num_tokens": 28967004.0,
      "reward": 0.26902028620243074,
      "reward_std": 0.26705058515071867,
      "rewards/ADEnReward/mean": 0.030168159306049346,
      "rewards/ADEnReward/std": 0.07774421945214272,
      "rewards/CriticV2Reward/mean": 0.1463521108031273,
      "rewards/CriticV2Reward/std": 0.24327176213264465,
      "rewards/StrictFormatReward/mean": 0.925000011920929,
      "rewards/StrictFormatReward/std": 0.34017834067344666,
      "sampling/importance_sampling_ratio/max": 2.629823637008667,
      "sampling/importance_sampling_ratio/mean": 0.41424151659011843,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.1681773185729982,
      "sampling/sampling_logp_difference/mean": 0.026383167132735252,
      "step": 260,
      "step_time": 14.161155000329018
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 819.4,
      "completions/max_terminated_length": 819.4,
      "completions/mean_length": 730.5958618164062,
      "completions/mean_terminated_length": 730.5958618164062,
      "completions/min_length": 657.2,
      "completions/min_terminated_length": 657.2,
      "entropy": 0.6704589664936066,
      "epoch": 0.039753975397539755,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.921875,
      "learning_rate": 3.6690647482014385e-07,
      "loss": -0.0642,
      "num_tokens": 29523163.0,
      "reward": 0.27605919539928436,
      "reward_std": 0.25954122841358185,
      "rewards/ADEnReward/mean": 0.031233209744095804,
      "rewards/ADEnReward/std": 0.08471837416291236,
      "rewards/CriticV2Reward/mean": 0.14815930277109146,
      "rewards/CriticV2Reward/std": 0.23515858948230745,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.19624642729759217,
      "sampling/importance_sampling_ratio/max": 2.5922812461853026,
      "sampling/importance_sampling_ratio/mean": 0.3501934826374054,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.589424896240234,
      "sampling/sampling_logp_difference/mean": 0.027005475759506226,
      "step": 265,
      "step_time": 13.391926497220993
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 817.6,
      "completions/max_terminated_length": 817.6,
      "completions/mean_length": 729.383349609375,
      "completions/mean_terminated_length": 729.383349609375,
      "completions/min_length": 662.0,
      "completions/min_terminated_length": 662.0,
      "entropy": 0.6677271962165833,
      "epoch": 0.0405040504050405,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3828125,
      "learning_rate": 3.54916067146283e-07,
      "loss": -0.0447,
      "num_tokens": 30079095.0,
      "reward": 0.2694405734539032,
      "reward_std": 0.27749488353729246,
      "rewards/ADEnReward/mean": 0.03271749503910541,
      "rewards/ADEnReward/std": 0.06845742091536522,
      "rewards/CriticV2Reward/mean": 0.141723071038723,
      "rewards/CriticV2Reward/std": 0.25534985661506654,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.23636061549186707,
      "sampling/importance_sampling_ratio/max": 2.580633020401001,
      "sampling/importance_sampling_ratio/mean": 0.35976060628890993,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.106691312789917,
      "sampling/sampling_logp_difference/mean": 0.027051576972007753,
      "step": 270,
      "step_time": 13.430866013094782
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00833333358168602,
      "completions/max_length": 1107.4,
      "completions/max_terminated_length": 815.8,
      "completions/mean_length": 745.3666870117188,
      "completions/mean_terminated_length": 738.6911376953125,
      "completions/min_length": 663.6,
      "completions/min_terminated_length": 663.6,
      "entropy": 0.7063815534114838,
      "epoch": 0.041254125412541254,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1875,
      "learning_rate": 3.4292565947242203e-07,
      "loss": -0.0802,
      "num_tokens": 30638991.0,
      "reward": 0.23000751733779906,
      "reward_std": 0.20237331390380858,
      "rewards/ADEnReward/mean": 0.014900790061801672,
      "rewards/ADEnReward/std": 0.047509999200701716,
      "rewards/CriticV2Reward/mean": 0.12094003856182098,
      "rewards/CriticV2Reward/std": 0.19368057399988176,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.294095641374588,
      "sampling/importance_sampling_ratio/max": 2.152031898498535,
      "sampling/importance_sampling_ratio/mean": 0.31464290618896484,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.139720010757446,
      "sampling/sampling_logp_difference/mean": 0.028014734387397766,
      "step": 275,
      "step_time": 15.272126619890333
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 966.2,
      "completions/max_terminated_length": 820.0,
      "completions/mean_length": 737.6000244140625,
      "completions/mean_terminated_length": 734.2439819335938,
      "completions/min_length": 667.0,
      "completions/min_terminated_length": 667.0,
      "entropy": 0.6556420803070069,
      "epoch": 0.04200420042004201,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.53125,
      "learning_rate": 3.309352517985611e-07,
      "loss": 0.0064,
      "num_tokens": 31197103.0,
      "reward": 0.28544660210609435,
      "reward_std": 0.2403573751449585,
      "rewards/ADEnReward/mean": 0.038210281357169154,
      "rewards/ADEnReward/std": 0.08102711364626884,
      "rewards/CriticV2Reward/mean": 0.1522363118827343,
      "rewards/CriticV2Reward/std": 0.21875708252191545,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.23636061549186707,
      "sampling/importance_sampling_ratio/max": 2.7030261993408202,
      "sampling/importance_sampling_ratio/mean": 0.41297419667243956,
      "sampling/importance_sampling_ratio/min": 2.0399140339577572e-06,
      "sampling/sampling_logp_difference/max": 3.274674487113953,
      "sampling/sampling_logp_difference/mean": 0.027079901099205016,
      "step": 280,
      "step_time": 14.378617286682129
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 936.2,
      "completions/max_terminated_length": 936.2,
      "completions/mean_length": 737.020849609375,
      "completions/mean_terminated_length": 737.020849609375,
      "completions/min_length": 661.4,
      "completions/min_terminated_length": 661.4,
      "entropy": 0.6838120698928833,
      "epoch": 0.04275427542754275,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.734375,
      "learning_rate": 3.1894484412470026e-07,
      "loss": -0.0648,
      "num_tokens": 31755108.0,
      "reward": 0.25459670424461367,
      "reward_std": 0.23896411657333375,
      "rewards/ADEnReward/mean": 0.022891263756901026,
      "rewards/ADEnReward/std": 0.04410543087869882,
      "rewards/CriticV2Reward/mean": 0.14253875762224197,
      "rewards/CriticV2Reward/std": 0.2287704974412918,
      "rewards/StrictFormatReward/mean": 0.8916666746139527,
      "rewards/StrictFormatReward/std": 0.3999585211277008,
      "sampling/importance_sampling_ratio/max": 2.5413411378860475,
      "sampling/importance_sampling_ratio/mean": 0.3536586403846741,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.051635646820069,
      "sampling/sampling_logp_difference/mean": 0.027624154463410378,
      "step": 285,
      "step_time": 14.16935004722327
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 810.4,
      "completions/max_terminated_length": 810.4,
      "completions/mean_length": 724.9750244140625,
      "completions/mean_terminated_length": 724.9750244140625,
      "completions/min_length": 626.0,
      "completions/min_terminated_length": 626.0,
      "entropy": 0.6491694033145905,
      "epoch": 0.043504350435043505,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.109375,
      "learning_rate": 3.069544364508393e-07,
      "loss": -0.0165,
      "num_tokens": 32309934.0,
      "reward": 0.26752397418022156,
      "reward_std": 0.22953160405158995,
      "rewards/ADEnReward/mean": 0.03401634339243174,
      "rewards/ADEnReward/std": 0.06949230469763279,
      "rewards/CriticV2Reward/mean": 0.13684094697237015,
      "rewards/CriticV2Reward/std": 0.21767082214355468,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.19624642729759217,
      "sampling/importance_sampling_ratio/max": 2.357547092437744,
      "sampling/importance_sampling_ratio/mean": 0.40171497464179995,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.093617391586304,
      "sampling/sampling_logp_difference/mean": 0.026295433193445204,
      "step": 290,
      "step_time": 13.348791869916022
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 797.8,
      "completions/max_terminated_length": 797.8,
      "completions/mean_length": 727.5416870117188,
      "completions/mean_terminated_length": 727.5416870117188,
      "completions/min_length": 673.6,
      "completions/min_terminated_length": 673.6,
      "entropy": 0.6584029793739319,
      "epoch": 0.04425442544254425,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.9296875,
      "learning_rate": 2.949640287769784e-07,
      "loss": -0.0725,
      "num_tokens": 32865616.0,
      "reward": 0.3513562619686127,
      "reward_std": 0.32367265224456787,
      "rewards/ADEnReward/mean": 0.050219583325088024,
      "rewards/ADEnReward/std": 0.0958492573350668,
      "rewards/CriticV2Reward/mean": 0.20196999311447145,
      "rewards/CriticV2Reward/std": 0.2919680178165436,
      "rewards/StrictFormatReward/mean": 0.9916666626930237,
      "rewards/StrictFormatReward/std": 0.05773502588272095,
      "sampling/importance_sampling_ratio/max": 2.3279226541519167,
      "sampling/importance_sampling_ratio/mean": 0.39847335815429685,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.63114709854126,
      "sampling/sampling_logp_difference/mean": 0.026548907533288,
      "step": 295,
      "step_time": 13.368336435966194
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1089.0,
      "completions/max_terminated_length": 940.2,
      "completions/mean_length": 737.7500122070312,
      "completions/mean_terminated_length": 734.3935302734375,
      "completions/min_length": 662.2,
      "completions/min_terminated_length": 662.2,
      "entropy": 0.6550628364086151,
      "epoch": 0.045004500450045004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.61328125,
      "learning_rate": 2.8297362110311753e-07,
      "loss": 0.035,
      "num_tokens": 33423668.0,
      "reward": 0.27340781688690186,
      "reward_std": 0.25819830000400545,
      "rewards/ADEnReward/mean": 0.028038720414042474,
      "rewards/ADEnReward/std": 0.06851784735918046,
      "rewards/CriticV2Reward/mean": 0.15370242297649384,
      "rewards/CriticV2Reward/std": 0.23523760139942168,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.34808469414710996,
      "sampling/importance_sampling_ratio/max": 2.835608148574829,
      "sampling/importance_sampling_ratio/mean": 0.45533260703086853,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.564956688880921,
      "sampling/sampling_logp_difference/mean": 0.02656121402978897,
      "step": 300,
      "step_time": 15.144700674340129
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 798.0,
      "completions/max_terminated_length": 798.0,
      "completions/mean_length": 733.9666870117187,
      "completions/mean_terminated_length": 733.9666870117187,
      "completions/min_length": 669.2,
      "completions/min_terminated_length": 669.2,
      "entropy": 0.6536436915397644,
      "epoch": 0.04575457545754576,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.8203125,
      "learning_rate": 2.7098321342925657e-07,
      "loss": -0.0131,
      "num_tokens": 33981308.0,
      "reward": 0.2970129132270813,
      "reward_std": 0.2672927647829056,
      "rewards/ADEnReward/mean": 0.016280055139213802,
      "rewards/ADEnReward/std": 0.04426044039428234,
      "rewards/CriticV2Reward/mean": 0.18656616508960724,
      "rewards/CriticV2Reward/std": 0.2617345839738846,
      "rewards/StrictFormatReward/mean": 0.9416666746139526,
      "rewards/StrictFormatReward/std": 0.2594019651412964,
      "sampling/importance_sampling_ratio/max": 2.434844732284546,
      "sampling/importance_sampling_ratio/mean": 0.3584721446037292,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.2410639762878417,
      "sampling/sampling_logp_difference/mean": 0.027303647249937057,
      "step": 305,
      "step_time": 13.214909846894443
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 946.2,
      "completions/max_terminated_length": 801.4,
      "completions/mean_length": 729.5666748046875,
      "completions/mean_terminated_length": 726.2310302734375,
      "completions/min_length": 666.0,
      "completions/min_terminated_length": 666.0,
      "entropy": 0.6449627339839935,
      "epoch": 0.0465046504650465,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.462890625,
      "learning_rate": 2.5899280575539566e-07,
      "loss": 0.0087,
      "num_tokens": 34537204.0,
      "reward": 0.27850418388843534,
      "reward_std": 0.2546063870191574,
      "rewards/ADEnReward/mean": 0.03761259010061622,
      "rewards/ADEnReward/std": 0.07763511538505555,
      "rewards/CriticV2Reward/mean": 0.14589156806468964,
      "rewards/CriticV2Reward/std": 0.23430478870868682,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.27702280282974245,
      "sampling/importance_sampling_ratio/max": 2.2312597751617433,
      "sampling/importance_sampling_ratio/mean": 0.3535082578659058,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.5829816341400145,
      "sampling/sampling_logp_difference/mean": 0.026253899559378624,
      "step": 310,
      "step_time": 14.251038361247629
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1111.2,
      "completions/max_terminated_length": 961.2,
      "completions/mean_length": 731.0833618164063,
      "completions/mean_terminated_length": 727.6387573242188,
      "completions/min_length": 425.6,
      "completions/min_terminated_length": 425.6,
      "entropy": 0.6603303372859954,
      "epoch": 0.047254725472547256,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.404296875,
      "learning_rate": 2.4700239808153475e-07,
      "loss": -0.0003,
      "num_tokens": 35093256.0,
      "reward": 0.3126928746700287,
      "reward_std": 0.2894691824913025,
      "rewards/ADEnReward/mean": 0.031799875758588315,
      "rewards/ADEnReward/std": 0.07806130312383175,
      "rewards/CriticV2Reward/mean": 0.18839299976825713,
      "rewards/CriticV2Reward/std": 0.2632020592689514,
      "rewards/StrictFormatReward/mean": 0.925,
      "rewards/StrictFormatReward/std": 0.3342098295688629,
      "sampling/importance_sampling_ratio/max": 2.1868972778320312,
      "sampling/importance_sampling_ratio/mean": 0.3813866376876831,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.841550922393799,
      "sampling/sampling_logp_difference/mean": 0.027124195545911788,
      "step": 315,
      "step_time": 15.209461131505668
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 834.2,
      "completions/max_terminated_length": 834.2,
      "completions/mean_length": 731.2291870117188,
      "completions/mean_terminated_length": 731.2291870117188,
      "completions/min_length": 632.0,
      "completions/min_terminated_length": 632.0,
      "entropy": 0.6608487665653229,
      "epoch": 0.048004800480048,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.67578125,
      "learning_rate": 2.3501199040767386e-07,
      "loss": -0.0772,
      "num_tokens": 35649903.0,
      "reward": 0.3282505989074707,
      "reward_std": 0.2977394014596939,
      "rewards/ADEnReward/mean": 0.03128300998359919,
      "rewards/ADEnReward/std": 0.061178040504455564,
      "rewards/CriticV2Reward/mean": 0.2028009206056595,
      "rewards/CriticV2Reward/std": 0.2806028872728348,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.3347578287124634,
      "sampling/importance_sampling_ratio/max": 2.501104164123535,
      "sampling/importance_sampling_ratio/mean": 0.42977511286735537,
      "sampling/importance_sampling_ratio/min": 0.0003163336776196957,
      "sampling/sampling_logp_difference/max": 3.5600287437438967,
      "sampling/sampling_logp_difference/mean": 0.02700127139687538,
      "step": 320,
      "step_time": 13.359743607882411
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 818.2,
      "completions/max_terminated_length": 818.2,
      "completions/mean_length": 733.5458374023438,
      "completions/mean_terminated_length": 733.5458374023438,
      "completions/min_length": 652.6,
      "completions/min_terminated_length": 652.6,
      "entropy": 0.688176554441452,
      "epoch": 0.048754875487548754,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.302734375,
      "learning_rate": 2.2302158273381293e-07,
      "loss": 0.053,
      "num_tokens": 36206706.0,
      "reward": 0.24143614768981933,
      "reward_std": 0.2442352443933487,
      "rewards/ADEnReward/mean": 0.02315669395029545,
      "rewards/ADEnReward/std": 0.06468662098050118,
      "rewards/CriticV2Reward/mean": 0.1241127759218216,
      "rewards/CriticV2Reward/std": 0.21812452077865602,
      "rewards/StrictFormatReward/mean": 0.9416666626930237,
      "rewards/StrictFormatReward/std": 0.25023548007011415,
      "sampling/importance_sampling_ratio/max": 2.460072135925293,
      "sampling/importance_sampling_ratio/mean": 0.4378948867321014,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.8208963990211486,
      "sampling/sampling_logp_difference/mean": 0.0267129085958004,
      "step": 325,
      "step_time": 13.40539922695607
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 849.6,
      "completions/max_terminated_length": 849.6,
      "completions/mean_length": 736.9750366210938,
      "completions/mean_terminated_length": 736.9750366210938,
      "completions/min_length": 552.2,
      "completions/min_terminated_length": 552.2,
      "entropy": 0.6794994473457336,
      "epoch": 0.04950495049504951,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.046875,
      "learning_rate": 2.1103117505995202e-07,
      "loss": -0.022,
      "num_tokens": 36764908.0,
      "reward": 0.33096388876438143,
      "reward_std": 0.29873789846897125,
      "rewards/ADEnReward/mean": 0.029305444657802583,
      "rewards/ADEnReward/std": 0.07149080038070679,
      "rewards/CriticV2Reward/mean": 0.20999177694320678,
      "rewards/CriticV2Reward/std": 0.275063157081604,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.3074225068092346,
      "sampling/importance_sampling_ratio/max": 2.370104718208313,
      "sampling/importance_sampling_ratio/mean": 0.3515215128660202,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 5.771962738037109,
      "sampling/sampling_logp_difference/mean": 0.028216013684868813,
      "step": 330,
      "step_time": 13.547085477039218
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 909.0,
      "completions/max_terminated_length": 909.0,
      "completions/mean_length": 736.2750122070313,
      "completions/mean_terminated_length": 736.2750122070313,
      "completions/min_length": 595.6,
      "completions/min_terminated_length": 595.6,
      "entropy": 0.6611269891262055,
      "epoch": 0.05025502550255025,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.94921875,
      "learning_rate": 1.9904076738609113e-07,
      "loss": -0.0151,
      "num_tokens": 37322686.0,
      "reward": 0.3241033911705017,
      "reward_std": 0.2873489558696747,
      "rewards/ADEnReward/mean": 0.031790857389569285,
      "rewards/ADEnReward/std": 0.07606348469853401,
      "rewards/CriticV2Reward/mean": 0.19981251955032348,
      "rewards/CriticV2Reward/std": 0.2744706034660339,
      "rewards/StrictFormatReward/mean": 0.925,
      "rewards/StrictFormatReward/std": 0.37487201690673827,
      "sampling/importance_sampling_ratio/max": 2.115146613121033,
      "sampling/importance_sampling_ratio/mean": 0.34482905715703965,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.7548720836639404,
      "sampling/sampling_logp_difference/mean": 0.027143753319978713,
      "step": 335,
      "step_time": 13.92563083646819
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1094.2,
      "completions/max_terminated_length": 943.2,
      "completions/mean_length": 740.1625244140625,
      "completions/mean_terminated_length": 736.7945190429688,
      "completions/min_length": 648.0,
      "completions/min_terminated_length": 648.0,
      "entropy": 0.6796614170074463,
      "epoch": 0.051005100510051006,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.3203125,
      "learning_rate": 1.870503597122302e-07,
      "loss": -0.0118,
      "num_tokens": 37881477.0,
      "reward": 0.2684012562036514,
      "reward_std": 0.2399124890565872,
      "rewards/ADEnReward/mean": 0.022319978382438422,
      "rewards/ADEnReward/std": 0.05877099372446537,
      "rewards/CriticV2Reward/mean": 0.15108125582337378,
      "rewards/CriticV2Reward/std": 0.22448689192533494,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.23636061549186707,
      "sampling/importance_sampling_ratio/max": 2.3924092769622805,
      "sampling/importance_sampling_ratio/mean": 0.323573100566864,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 4.640535640716553,
      "sampling/sampling_logp_difference/mean": 0.027076425030827522,
      "step": 340,
      "step_time": 15.11005504000932
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 902.8,
      "completions/max_terminated_length": 902.8,
      "completions/mean_length": 733.1625244140625,
      "completions/mean_terminated_length": 733.1625244140625,
      "completions/min_length": 675.4,
      "completions/min_terminated_length": 675.4,
      "entropy": 0.6722322583198548,
      "epoch": 0.05175517551755175,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.62890625,
      "learning_rate": 1.750599520383693e-07,
      "loss": 0.0122,
      "num_tokens": 38438508.0,
      "reward": 0.2832422822713852,
      "reward_std": 0.2662737250328064,
      "rewards/ADEnReward/mean": 0.01820658575743437,
      "rewards/ADEnReward/std": 0.051234375685453415,
      "rewards/CriticV2Reward/mean": 0.16503570079803467,
      "rewards/CriticV2Reward/std": 0.25538637936115266,
      "rewards/StrictFormatReward/mean": 1.0,
      "rewards/StrictFormatReward/std": 0.0,
      "sampling/importance_sampling_ratio/max": 2.2767002105712892,
      "sampling/importance_sampling_ratio/mean": 0.4129745185375214,
      "sampling/importance_sampling_ratio/min": 0.0010848915204405784,
      "sampling/sampling_logp_difference/max": 1.5775320053100585,
      "sampling/sampling_logp_difference/mean": 0.0266124464571476,
      "step": 345,
      "step_time": 13.816253677289932
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 795.6,
      "completions/max_terminated_length": 795.6,
      "completions/mean_length": 726.17919921875,
      "completions/mean_terminated_length": 726.17919921875,
      "completions/min_length": 548.2,
      "completions/min_terminated_length": 548.2,
      "entropy": 0.6423774480819702,
      "epoch": 0.052505250525052505,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.427734375,
      "learning_rate": 1.630695443645084e-07,
      "loss": -0.0081,
      "num_tokens": 38993991.0,
      "reward": 0.3389123469591141,
      "reward_std": 0.29732525944709776,
      "rewards/ADEnReward/mean": 0.04917324259877205,
      "rewards/ADEnReward/std": 0.09704692177474498,
      "rewards/CriticV2Reward/mean": 0.19807242900133132,
      "rewards/CriticV2Reward/std": 0.2688820421695709,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.39194485545158386,
      "sampling/importance_sampling_ratio/max": 2.469262218475342,
      "sampling/importance_sampling_ratio/mean": 0.41353853344917296,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.8893362045288087,
      "sampling/sampling_logp_difference/mean": 0.026003163307905197,
      "step": 350,
      "step_time": 13.209398588072508
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 952.4,
      "completions/max_terminated_length": 811.4,
      "completions/mean_length": 732.6875244140625,
      "completions/mean_terminated_length": 729.3064819335938,
      "completions/min_length": 670.4,
      "completions/min_terminated_length": 670.4,
      "entropy": 0.6782492220401763,
      "epoch": 0.05325532553255326,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.265625,
      "learning_rate": 1.5107913669064747e-07,
      "loss": -0.0119,
      "num_tokens": 39550364.0,
      "reward": 0.24834446907043456,
      "reward_std": 0.22241731882095336,
      "rewards/ADEnReward/mean": 0.018331396393477917,
      "rewards/ADEnReward/std": 0.049924323707818984,
      "rewards/CriticV2Reward/mean": 0.13501306772232055,
      "rewards/CriticV2Reward/std": 0.21232716143131256,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.23636061549186707,
      "sampling/importance_sampling_ratio/max": 2.49073007106781,
      "sampling/importance_sampling_ratio/mean": 0.37095364928245544,
      "sampling/importance_sampling_ratio/min": 0.0006488471291959286,
      "sampling/sampling_logp_difference/max": 2.1764797449111937,
      "sampling/sampling_logp_difference/mean": 0.02676672637462616,
      "step": 355,
      "step_time": 14.224239609390498
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 1073.6,
      "completions/max_terminated_length": 925.2,
      "completions/mean_length": 734.620849609375,
      "completions/mean_terminated_length": 731.280419921875,
      "completions/min_length": 437.4,
      "completions/min_terminated_length": 437.4,
      "entropy": 0.6800269901752471,
      "epoch": 0.054005400540054004,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.455078125,
      "learning_rate": 1.3908872901678656e-07,
      "loss": 0.0046,
      "num_tokens": 40107809.0,
      "reward": 0.24553753435611725,
      "reward_std": 0.21728258728981018,
      "rewards/ADEnReward/mean": 0.02120685214176774,
      "rewards/ADEnReward/std": 0.05041029825806618,
      "rewards/CriticV2Reward/mean": 0.13183067739009857,
      "rewards/CriticV2Reward/std": 0.2057559758424759,
      "rewards/StrictFormatReward/mean": 0.925000011920929,
      "rewards/StrictFormatReward/std": 0.34017834067344666,
      "sampling/importance_sampling_ratio/max": 2.6524218559265136,
      "sampling/importance_sampling_ratio/mean": 0.4359212100505829,
      "sampling/importance_sampling_ratio/min": 0.00012990221148356795,
      "sampling/sampling_logp_difference/max": 4.0367347240448,
      "sampling/sampling_logp_difference/mean": 0.02658579796552658,
      "step": 360,
      "step_time": 15.101525206118822
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 807.2,
      "completions/max_terminated_length": 807.2,
      "completions/mean_length": 728.8291870117188,
      "completions/mean_terminated_length": 728.8291870117188,
      "completions/min_length": 439.6,
      "completions/min_terminated_length": 439.6,
      "entropy": 0.6668968260288238,
      "epoch": 0.054755475547554756,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 1.1171875,
      "learning_rate": 1.2709832134292565e-07,
      "loss": -0.0227,
      "num_tokens": 40663784.0,
      "reward": 0.29526926279067994,
      "reward_std": 0.2813252925872803,
      "rewards/ADEnReward/mean": 0.03403082452714443,
      "rewards/ADEnReward/std": 0.08168720752000809,
      "rewards/CriticV2Reward/mean": 0.16957174986600876,
      "rewards/CriticV2Reward/std": 0.24872368276119233,
      "rewards/StrictFormatReward/mean": 0.9166666626930237,
      "rewards/StrictFormatReward/std": 0.34808469414710996,
      "sampling/importance_sampling_ratio/max": 2.390850281715393,
      "sampling/importance_sampling_ratio/mean": 0.4092385768890381,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.3626219749450685,
      "sampling/sampling_logp_difference/mean": 0.026888956502079963,
      "step": 365,
      "step_time": 13.315052114892751
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 804.2,
      "completions/max_terminated_length": 804.2,
      "completions/mean_length": 733.4416870117187,
      "completions/mean_terminated_length": 733.4416870117187,
      "completions/min_length": 663.8,
      "completions/min_terminated_length": 663.8,
      "entropy": 0.6606446027755737,
      "epoch": 0.0555055505550555,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.78125,
      "learning_rate": 1.1510791366906475e-07,
      "loss": -0.0329,
      "num_tokens": 41220978.0,
      "reward": 0.2930675059556961,
      "reward_std": 0.2800747185945511,
      "rewards/ADEnReward/mean": 0.03040240630507469,
      "rewards/ADEnReward/std": 0.07180959656834603,
      "rewards/CriticV2Reward/mean": 0.16849841624498368,
      "rewards/CriticV2Reward/std": 0.25502766370773317,
      "rewards/StrictFormatReward/mean": 0.9416666746139526,
      "rewards/StrictFormatReward/std": 0.30006415247917173,
      "sampling/importance_sampling_ratio/max": 2.7124672889709474,
      "sampling/importance_sampling_ratio/mean": 0.43735997676849364,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.8521456480026246,
      "sampling/sampling_logp_difference/mean": 0.02668215483427048,
      "step": 370,
      "step_time": 13.217574856709689
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 803.4,
      "completions/max_terminated_length": 803.4,
      "completions/mean_length": 732.1750244140625,
      "completions/mean_terminated_length": 732.1750244140625,
      "completions/min_length": 664.8,
      "completions/min_terminated_length": 664.8,
      "entropy": 0.6729780435562134,
      "epoch": 0.056255625562556255,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.6875,
      "learning_rate": 1.0311750599520383e-07,
      "loss": 0.057,
      "num_tokens": 41777820.0,
      "reward": 0.292266783118248,
      "reward_std": 0.2692907273769379,
      "rewards/ADEnReward/mean": 0.02988054435700178,
      "rewards/ADEnReward/std": 0.0724082849919796,
      "rewards/CriticV2Reward/mean": 0.16571956276893615,
      "rewards/CriticV2Reward/std": 0.25571154057979584,
      "rewards/StrictFormatReward/mean": 0.9666666626930237,
      "rewards/StrictFormatReward/std": 0.19624642729759217,
      "sampling/importance_sampling_ratio/max": 2.654768705368042,
      "sampling/importance_sampling_ratio/mean": 0.4235609233379364,
      "sampling/importance_sampling_ratio/min": 0.000751539459452033,
      "sampling/sampling_logp_difference/max": 3.4154165744781495,
      "sampling/sampling_logp_difference/mean": 0.027747052907943725,
      "step": 375,
      "step_time": 13.223657737579197
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 946.8,
      "completions/max_terminated_length": 946.8,
      "completions/mean_length": 729.4666870117187,
      "completions/mean_terminated_length": 729.4666870117187,
      "completions/min_length": 556.4,
      "completions/min_terminated_length": 556.4,
      "entropy": 0.6680528521537781,
      "epoch": 0.05700570057005701,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.90234375,
      "learning_rate": 9.112709832134293e-08,
      "loss": 0.0034,
      "num_tokens": 42333820.0,
      "reward": 0.30099524557590485,
      "reward_std": 0.28950676918029783,
      "rewards/ADEnReward/mean": 0.04762665368616581,
      "rewards/ADEnReward/std": 0.11501347124576569,
      "rewards/CriticV2Reward/mean": 0.16253525763750076,
      "rewards/CriticV2Reward/std": 0.23468865156173707,
      "rewards/StrictFormatReward/mean": 0.9083333373069763,
      "rewards/StrictFormatReward/std": 0.4005065202713013,
      "sampling/importance_sampling_ratio/max": 2.560587167739868,
      "sampling/importance_sampling_ratio/mean": 0.37538244128227233,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.4054880142211914,
      "sampling/sampling_logp_difference/mean": 0.02632361464202404,
      "step": 380,
      "step_time": 14.30399916516617
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 832.2,
      "completions/max_terminated_length": 832.2,
      "completions/mean_length": 735.6333618164062,
      "completions/mean_terminated_length": 735.6333618164062,
      "completions/min_length": 652.8,
      "completions/min_terminated_length": 652.8,
      "entropy": 0.6726574301719666,
      "epoch": 0.057755775577557754,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.609375,
      "learning_rate": 7.913669064748201e-08,
      "loss": -0.048,
      "num_tokens": 42891412.0,
      "reward": 0.30219454765319825,
      "reward_std": 0.27768056392669677,
      "rewards/ADEnReward/mean": 0.03429584912955761,
      "rewards/ADEnReward/std": 0.07258765399456024,
      "rewards/CriticV2Reward/mean": 0.17539868950843812,
      "rewards/CriticV2Reward/std": 0.2650424838066101,
      "rewards/StrictFormatReward/mean": 0.925000011920929,
      "rewards/StrictFormatReward/std": 0.34017834067344666,
      "sampling/importance_sampling_ratio/max": 2.453314113616943,
      "sampling/importance_sampling_ratio/mean": 0.3553328990936279,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.31168155670166,
      "sampling/sampling_logp_difference/mean": 0.02763594649732113,
      "step": 385,
      "step_time": 13.449316807556897
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 947.2,
      "completions/max_terminated_length": 947.2,
      "completions/mean_length": 739.39169921875,
      "completions/mean_terminated_length": 739.39169921875,
      "completions/min_length": 671.8,
      "completions/min_terminated_length": 671.8,
      "entropy": 0.6789502263069153,
      "epoch": 0.05850585058505851,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.474609375,
      "learning_rate": 6.71462829736211e-08,
      "loss": -0.0058,
      "num_tokens": 43450082.0,
      "reward": 0.2598124861717224,
      "reward_std": 0.2435460090637207,
      "rewards/ADEnReward/mean": 0.04101892188191414,
      "rewards/ADEnReward/std": 0.09139346182346345,
      "rewards/CriticV2Reward/mean": 0.12296021729707718,
      "rewards/CriticV2Reward/std": 0.21000217646360397,
      "rewards/StrictFormatReward/mean": 0.9583333373069763,
      "rewards/StrictFormatReward/std": 0.21928777694702148,
      "sampling/importance_sampling_ratio/max": 2.6636743545532227,
      "sampling/importance_sampling_ratio/mean": 0.41307313442230226,
      "sampling/importance_sampling_ratio/min": 0.00033994775731116533,
      "sampling/sampling_logp_difference/max": 4.322899675369262,
      "sampling/sampling_logp_difference/mean": 0.026649556308984756,
      "step": 390,
      "step_time": 14.210391772259026
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 821.2,
      "completions/max_terminated_length": 821.2,
      "completions/mean_length": 729.7666870117188,
      "completions/mean_terminated_length": 729.7666870117188,
      "completions/min_length": 554.6,
      "completions/min_terminated_length": 554.6,
      "entropy": 0.6675756096839904,
      "epoch": 0.05925592559255925,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.671875,
      "learning_rate": 5.5155875299760186e-08,
      "loss": 0.0211,
      "num_tokens": 44006314.0,
      "reward": 0.28109568953514097,
      "reward_std": 0.25426478683948517,
      "rewards/ADEnReward/mean": 0.01611993897240609,
      "rewards/ADEnReward/std": 0.039170157397165895,
      "rewards/CriticV2Reward/mean": 0.16997574269771576,
      "rewards/CriticV2Reward/std": 0.24328334629535675,
      "rewards/StrictFormatReward/mean": 0.949999988079071,
      "rewards/StrictFormatReward/std": 0.31171647906303407,
      "sampling/importance_sampling_ratio/max": 2.653896713256836,
      "sampling/importance_sampling_ratio/mean": 0.4388413727283478,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 1.7347460508346557,
      "sampling/sampling_logp_difference/mean": 0.027177633345127107,
      "step": 395,
      "step_time": 13.348359900712968
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00833333358168602,
      "completions/max_length": 1113.0,
      "completions/max_terminated_length": 824.8,
      "completions/mean_length": 737.6625244140625,
      "completions/mean_terminated_length": 730.9420166015625,
      "completions/min_length": 427.0,
      "completions/min_terminated_length": 427.0,
      "entropy": 0.6727102100849152,
      "epoch": 0.060006000600060005,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.451171875,
      "learning_rate": 4.316546762589928e-08,
      "loss": -0.0499,
      "num_tokens": 44564569.0,
      "reward": 0.31804165840148924,
      "reward_std": 0.26383175849914553,
      "rewards/ADEnReward/mean": 0.0430668905377388,
      "rewards/ADEnReward/std": 0.08763031214475632,
      "rewards/CriticV2Reward/mean": 0.18247474879026412,
      "rewards/CriticV2Reward/std": 0.24957966804504395,
      "rewards/StrictFormatReward/mean": 0.925,
      "rewards/StrictFormatReward/std": 0.37487201690673827,
      "sampling/importance_sampling_ratio/max": 2.1602240800857544,
      "sampling/importance_sampling_ratio/mean": 0.3833409368991852,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.4215245723724363,
      "sampling/sampling_logp_difference/mean": 0.02680293843150139,
      "step": 400,
      "step_time": 15.337327389977872
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 806.8,
      "completions/max_terminated_length": 806.8,
      "completions/mean_length": 733.6291748046875,
      "completions/mean_terminated_length": 733.6291748046875,
      "completions/min_length": 655.4,
      "completions/min_terminated_length": 655.4,
      "entropy": 0.6855071008205413,
      "epoch": 0.06075607560756076,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.89453125,
      "learning_rate": 3.1175059952038366e-08,
      "loss": 0.0418,
      "num_tokens": 45121504.0,
      "reward": 0.30168403387069703,
      "reward_std": 0.26787208914756777,
      "rewards/ADEnReward/mean": 0.029562235996127128,
      "rewards/ADEnReward/std": 0.0694606777280569,
      "rewards/CriticV2Reward/mean": 0.17628844380378722,
      "rewards/CriticV2Reward/std": 0.2518341809511185,
      "rewards/StrictFormatReward/mean": 0.9583333373069763,
      "rewards/StrictFormatReward/std": 0.21928777694702148,
      "sampling/importance_sampling_ratio/max": 2.63645486831665,
      "sampling/importance_sampling_ratio/mean": 0.37098753452301025,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 2.670264649391174,
      "sampling/sampling_logp_difference/mean": 0.027659954503178596,
      "step": 405,
      "step_time": 13.189621604420244
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.00416666679084301,
      "completions/max_length": 956.2,
      "completions/max_terminated_length": 810.6,
      "completions/mean_length": 735.0416870117188,
      "completions/mean_terminated_length": 731.6653564453125,
      "completions/min_length": 650.8,
      "completions/min_terminated_length": 650.8,
      "entropy": 0.6711634755134582,
      "epoch": 0.061506150615061504,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.73828125,
      "learning_rate": 1.9184652278177456e-08,
      "loss": -0.0293,
      "num_tokens": 45678922.0,
      "reward": 0.27163800299167634,
      "reward_std": 0.26216488778591157,
      "rewards/ADEnReward/mean": 0.014222659170627594,
      "rewards/ADEnReward/std": 0.03859034609049559,
      "rewards/CriticV2Reward/mean": 0.1624153345823288,
      "rewards/CriticV2Reward/std": 0.2478887140750885,
      "rewards/StrictFormatReward/mean": 0.95,
      "rewards/StrictFormatReward/std": 0.27702280282974245,
      "sampling/importance_sampling_ratio/max": 2.693353033065796,
      "sampling/importance_sampling_ratio/mean": 0.4223414957523346,
      "sampling/importance_sampling_ratio/min": 1.2648591791730722e-07,
      "sampling/sampling_logp_difference/max": 5.323451900482178,
      "sampling/sampling_logp_difference/mean": 0.026813267916440963,
      "step": 410,
      "step_time": 14.17747332798317
    },
    {
      "clip_ratio/high_max": 0.0,
      "clip_ratio/high_mean": 0.0,
      "clip_ratio/low_mean": 0.0,
      "clip_ratio/low_min": 0.0,
      "clip_ratio/region_mean": 0.0,
      "completions/clipped_ratio": 0.0,
      "completions/max_length": 1070.4,
      "completions/max_terminated_length": 1070.4,
      "completions/mean_length": 739.8833618164062,
      "completions/mean_terminated_length": 739.8833618164062,
      "completions/min_length": 544.0,
      "completions/min_terminated_length": 544.0,
      "entropy": 0.6690365135669708,
      "epoch": 0.06225622562256226,
      "frac_reward_zero_std": 0.0,
      "grad_norm": 0.703125,
      "learning_rate": 7.194244604316547e-09,
      "loss": -0.0379,
      "num_tokens": 46237678.0,
      "reward": 0.3154370367527008,
      "reward_std": 0.2778420358896255,
      "rewards/ADEnReward/mean": 0.033237920398823916,
      "rewards/ADEnReward/std": 0.0803776565939188,
      "rewards/CriticV2Reward/mean": 0.1888657659292221,
      "rewards/CriticV2Reward/std": 0.25663301944732664,
      "rewards/StrictFormatReward/mean": 0.9333333492279052,
      "rewards/StrictFormatReward/std": 0.32310550212860106,
      "sampling/importance_sampling_ratio/max": 2.434389925003052,
      "sampling/importance_sampling_ratio/mean": 0.3478035569190979,
      "sampling/importance_sampling_ratio/min": 0.0,
      "sampling/sampling_logp_difference/max": 3.7522983074188234,
      "sampling/sampling_logp_difference/mean": 0.027356555312871934,
      "step": 415,
      "step_time": 15.047657939139754
    }
  ],
  "logging_steps": 5,
  "max_steps": 417,
  "num_input_tokens_seen": 46460815,
  "num_train_epochs": 1,
  "save_steps": 100,
  "stateful_callbacks": {
    "TrainerControl": {
      "args": {
        "should_epoch_stop": false,
        "should_evaluate": false,
        "should_log": false,
        "should_save": true,
        "should_training_stop": true
      },
      "attributes": {}
    }
  },
  "total_flos": 0.0,
  "train_batch_size": 6,
  "trial_name": null,
  "trial_params": null
}