{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.06255625562556255, "eval_steps": 100, "global_step": 417, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 902.6, "completions/max_terminated_length": 902.6, "completions/mean_length": 736.9750244140625, "completions/mean_terminated_length": 736.9750244140625, "completions/min_length": 667.8, "completions/min_terminated_length": 667.8, "entropy": 0.6896837294101715, "epoch": 0.00075007500750075, "frac_reward_zero_std": 0.0, "grad_norm": 0.5546875, "learning_rate": 9.904076738609112e-07, "loss": -0.0561, "num_tokens": 557882.0, "reward": 0.6253534197807312, "reward_std": 0.21987640857696533, "rewards/ADEnReward/mean": 0.028631300292909147, "rewards/ADEnReward/std": 0.07471808642148972, "rewards/CriticEdgeReward/mean": 0.5025554478168488, "rewards/CriticEdgeReward/std": 0.19941229224205018, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.3347578287124634, "sampling/importance_sampling_ratio/max": 2.7144235610961913, "sampling/importance_sampling_ratio/mean": 0.4599367678165436, "sampling/importance_sampling_ratio/min": 1.4063954040466343e-06, "sampling/sampling_logp_difference/max": 5.391781187057495, "sampling/sampling_logp_difference/mean": 0.027573855966329573, "step": 5, "step_time": 21.64242088012397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1085.2, "completions/max_terminated_length": 1085.2, "completions/mean_length": 745.56669921875, "completions/mean_terminated_length": 745.56669921875, "completions/min_length": 674.8, "completions/min_terminated_length": 674.8, "entropy": 0.6767813801765442, "epoch": 0.0015001500150015, "frac_reward_zero_std": 0.0, "grad_norm": 0.85546875, "learning_rate": 9.784172661870503e-07, "loss": -0.0284, "num_tokens": 1118162.0, "reward": 0.6478345513343811, "reward_std": 0.19704576432704926, "rewards/ADEnReward/mean": 0.016783953923732043, "rewards/ADEnReward/std": 0.04695849996060133, "rewards/CriticEdgeReward/mean": 0.5377172410488129, "rewards/CriticEdgeReward/std": 0.18845856487751006, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3079705059528351, "sampling/importance_sampling_ratio/max": 2.4997782945632934, "sampling/importance_sampling_ratio/mean": 0.41286763548851013, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1890854835510254, "sampling/sampling_logp_difference/mean": 0.027684328705072404, "step": 10, "step_time": 19.549452285375445 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 814.8, "completions/max_terminated_length": 814.8, "completions/mean_length": 732.1041870117188, "completions/mean_terminated_length": 732.1041870117188, "completions/min_length": 678.0, "completions/min_terminated_length": 678.0, "entropy": 0.6601380050182343, "epoch": 0.0022502250225022503, "frac_reward_zero_std": 0.0, "grad_norm": 0.6953125, "learning_rate": 9.664268585131893e-07, "loss": 0.0654, "num_tokens": 1675019.0, "reward": 0.648806631565094, "reward_std": 0.24549749493598938, "rewards/ADEnReward/mean": 0.03518346101045609, "rewards/ADEnReward/std": 0.0901635468006134, "rewards/CriticEdgeReward/mean": 0.5161231279373169, "rewards/CriticEdgeReward/std": 0.2094112902879715, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.25063636302948, "sampling/importance_sampling_ratio/mean": 0.3282919883728027, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.486510181427002, "sampling/sampling_logp_difference/mean": 0.02701355367898941, "step": 15, "step_time": 17.75322556840256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 857.4, "completions/max_terminated_length": 857.4, "completions/mean_length": 732.058349609375, "completions/mean_terminated_length": 732.058349609375, "completions/min_length": 649.6, "completions/min_terminated_length": 649.6, "entropy": 0.6416908502578735, "epoch": 0.003000300030003, "frac_reward_zero_std": 0.0, "grad_norm": 0.828125, "learning_rate": 9.544364508393285e-07, "loss": -0.0123, "num_tokens": 2232121.0, "reward": 0.6910966277122498, "reward_std": 0.23999985456466674, "rewards/ADEnReward/mean": 0.02863377407193184, "rewards/ADEnReward/std": 0.0722395770251751, "rewards/CriticEdgeReward/mean": 0.5682961583137512, "rewards/CriticEdgeReward/std": 0.22044118046760558, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.307643675804138, "sampling/importance_sampling_ratio/mean": 0.40845474004745486, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.765939402580261, "sampling/sampling_logp_difference/mean": 0.027239738777279855, "step": 20, "step_time": 18.00340573452413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 861.6, "completions/max_terminated_length": 861.6, "completions/mean_length": 723.9416870117187, "completions/mean_terminated_length": 723.9416870117187, "completions/min_length": 542.2, "completions/min_terminated_length": 542.2, "entropy": 0.6595201253890991, "epoch": 0.0037503750375037503, "frac_reward_zero_std": 0.0, "grad_norm": 0.60546875, "learning_rate": 9.424460431654676e-07, "loss": 0.0051, "num_tokens": 2786811.0, "reward": 0.6198334574699402, "reward_std": 0.2572463184595108, "rewards/ADEnReward/mean": 0.03799758106470108, "rewards/ADEnReward/std": 0.0950001284480095, "rewards/CriticEdgeReward/mean": 0.4918358564376831, "rewards/CriticEdgeReward/std": 0.2063400000333786, "rewards/StrictFormatReward/mean": 0.899999988079071, "rewards/StrictFormatReward/std": 0.3790323734283447, "sampling/importance_sampling_ratio/max": 2.5853229761123657, "sampling/importance_sampling_ratio/mean": 0.4161765515804291, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.767236423492432, "sampling/sampling_logp_difference/mean": 0.025676552578806876, "step": 25, "step_time": 18.056161639466882 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1062.8, "completions/max_terminated_length": 1062.8, "completions/mean_length": 741.5083618164062, "completions/mean_terminated_length": 741.5083618164062, "completions/min_length": 657.8, "completions/min_terminated_length": 657.8, "entropy": 0.666927695274353, "epoch": 0.004500450045004501, "frac_reward_zero_std": 0.0, "grad_norm": 0.349609375, "learning_rate": 9.304556354916066e-07, "loss": -0.013, "num_tokens": 3345781.0, "reward": 0.6449725866317749, "reward_std": 0.22292991876602172, "rewards/ADEnReward/mean": 0.026060701720416544, "rewards/ADEnReward/std": 0.05949128344655037, "rewards/CriticEdgeReward/mean": 0.5264118611812592, "rewards/CriticEdgeReward/std": 0.20431106686592101, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.36890350580215453, "sampling/importance_sampling_ratio/max": 2.14269073009491, "sampling/importance_sampling_ratio/mean": 0.36652535796165464, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5075393438339235, "sampling/sampling_logp_difference/mean": 0.0269878089427948, "step": 30, "step_time": 19.31739601250738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 921.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 738.7041870117188, "completions/mean_terminated_length": 738.7041870117188, "completions/min_length": 678.2, "completions/min_terminated_length": 678.2, "entropy": 0.6612422406673432, "epoch": 0.005250525052505251, "frac_reward_zero_std": 0.0, "grad_norm": 0.9140625, "learning_rate": 9.184652278177458e-07, "loss": -0.0632, "num_tokens": 3904510.0, "reward": 0.6685188174247741, "reward_std": 0.22824054062366486, "rewards/ADEnReward/mean": 0.020993375219404697, "rewards/ADEnReward/std": 0.05384831465780735, "rewards/CriticEdgeReward/mean": 0.5516920864582062, "rewards/CriticEdgeReward/std": 0.21149038076400756, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.21331926584243774, "sampling/importance_sampling_ratio/max": 2.433179807662964, "sampling/importance_sampling_ratio/mean": 0.40854416489601136, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.970175457000733, "sampling/sampling_logp_difference/mean": 0.027108031511306762, "step": 35, "step_time": 18.358295069821178 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 877.0, "completions/max_terminated_length": 877.0, "completions/mean_length": 737.25419921875, "completions/mean_terminated_length": 737.25419921875, "completions/min_length": 563.2, "completions/min_terminated_length": 563.2, "entropy": 0.6781140625476837, "epoch": 0.006000600060006, "frac_reward_zero_std": 0.0, "grad_norm": 0.427734375, "learning_rate": 9.064748201438849e-07, "loss": -0.0404, "num_tokens": 4462651.0, "reward": 0.6330280423164367, "reward_std": 0.22336381375789643, "rewards/ADEnReward/mean": 0.02816481962800026, "rewards/ADEnReward/std": 0.06972450911998748, "rewards/CriticEdgeReward/mean": 0.5115298748016357, "rewards/CriticEdgeReward/std": 0.20885967314243317, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3111684799194336, "sampling/importance_sampling_ratio/max": 2.5332077026367186, "sampling/importance_sampling_ratio/mean": 0.348198401927948, "sampling/importance_sampling_ratio/min": 1.0409014794277028e-05, "sampling/sampling_logp_difference/max": 4.22596697807312, "sampling/sampling_logp_difference/mean": 0.027745723724365234, "step": 40, "step_time": 18.2379128087312 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 809.4, "completions/max_terminated_length": 809.4, "completions/mean_length": 731.7041870117188, "completions/mean_terminated_length": 731.7041870117188, "completions/min_length": 534.8, "completions/min_terminated_length": 534.8, "entropy": 0.679815000295639, "epoch": 0.0067506750675067504, "frac_reward_zero_std": 0.0, "grad_norm": 0.578125, "learning_rate": 8.944844124700239e-07, "loss": 0.0109, "num_tokens": 5019268.0, "reward": 0.6154009461402893, "reward_std": 0.22656834423542022, "rewards/ADEnReward/mean": 0.026610768772661685, "rewards/ADEnReward/std": 0.06118314117193222, "rewards/CriticEdgeReward/mean": 0.4962901473045349, "rewards/CriticEdgeReward/std": 0.2063582420349121, "rewards/StrictFormatReward/mean": 0.925000011920929, "rewards/StrictFormatReward/std": 0.34017834067344666, "sampling/importance_sampling_ratio/max": 2.6422813892364503, "sampling/importance_sampling_ratio/mean": 0.3912628054618835, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.1621949911117553, "sampling/sampling_logp_difference/mean": 0.027560533210635185, "step": 45, "step_time": 17.72976392423734 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 952.2, "completions/max_terminated_length": 807.8, "completions/mean_length": 733.7958374023438, "completions/mean_terminated_length": 730.512060546875, "completions/min_length": 656.4, "completions/min_terminated_length": 656.4, "entropy": 0.669914835691452, "epoch": 0.007500750075007501, "frac_reward_zero_std": 0.0, "grad_norm": 0.9453125, "learning_rate": 8.82494004796163e-07, "loss": -0.0235, "num_tokens": 5576451.0, "reward": 0.6434481382369995, "reward_std": 0.204806911945343, "rewards/ADEnReward/mean": 0.02442994313314557, "rewards/ADEnReward/std": 0.06445258595049382, "rewards/CriticEdgeReward/mean": 0.5256848335266113, "rewards/CriticEdgeReward/std": 0.18509471118450166, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.556506538391113, "sampling/importance_sampling_ratio/mean": 0.4481285572052002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.283991813659668, "sampling/sampling_logp_difference/mean": 0.026617782562971114, "step": 50, "step_time": 18.52655699821189 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 970.0, "completions/max_terminated_length": 820.8, "completions/mean_length": 739.0125122070312, "completions/mean_terminated_length": 735.6781005859375, "completions/min_length": 570.4, "completions/min_terminated_length": 570.4, "entropy": 0.674360454082489, "epoch": 0.00825082508250825, "frac_reward_zero_std": 0.0, "grad_norm": 0.478515625, "learning_rate": 8.705035971223021e-07, "loss": 0.0362, "num_tokens": 6135078.0, "reward": 0.6392877459526062, "reward_std": 0.2148994356393814, "rewards/ADEnReward/mean": 0.01358063705265522, "rewards/ADEnReward/std": 0.04260286279022694, "rewards/CriticEdgeReward/mean": 0.5298737525939942, "rewards/CriticEdgeReward/std": 0.19750311076641083, "rewards/StrictFormatReward/mean": 0.9583333373069763, "rewards/StrictFormatReward/std": 0.21928777694702148, "sampling/importance_sampling_ratio/max": 2.1741480588912965, "sampling/importance_sampling_ratio/mean": 0.3659940779209137, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.300462746620179, "sampling/sampling_logp_difference/mean": 0.028139141201972962, "step": 55, "step_time": 18.684299015067516 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 821.4, "completions/max_terminated_length": 821.4, "completions/mean_length": 730.2416870117188, "completions/mean_terminated_length": 730.2416870117188, "completions/min_length": 538.6, "completions/min_terminated_length": 538.6, "entropy": 0.6734500288963318, "epoch": 0.009000900090009001, "frac_reward_zero_std": 0.0, "grad_norm": 0.4375, "learning_rate": 8.585131894484412e-07, "loss": 0.0069, "num_tokens": 6691248.0, "reward": 0.6695833444595337, "reward_std": 0.24675742387771607, "rewards/ADEnReward/mean": 0.025426148204132915, "rewards/ADEnReward/std": 0.06754331290721893, "rewards/CriticEdgeReward/mean": 0.5483238458633423, "rewards/CriticEdgeReward/std": 0.21718314588069915, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.2539814531803131, "sampling/importance_sampling_ratio/max": 2.1626408815383913, "sampling/importance_sampling_ratio/mean": 0.3702266216278076, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.0395228385925295, "sampling/sampling_logp_difference/mean": 0.027444090321660043, "step": 60, "step_time": 17.866433706413954 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 922.2, "completions/max_terminated_length": 922.2, "completions/mean_length": 735.1666870117188, "completions/mean_terminated_length": 735.1666870117188, "completions/min_length": 647.4, "completions/min_terminated_length": 647.4, "entropy": 0.6681231558322906, "epoch": 0.00975097509750975, "frac_reward_zero_std": 0.0, "grad_norm": 0.43359375, "learning_rate": 8.465227817745802e-07, "loss": -0.0682, "num_tokens": 7248728.0, "reward": 0.6629689574241638, "reward_std": 0.20367246270179748, "rewards/ADEnReward/mean": 0.03383460873737931, "rewards/ADEnReward/std": 0.06237032376229763, "rewards/CriticEdgeReward/mean": 0.5349676430225372, "rewards/CriticEdgeReward/std": 0.19174161553382874, "rewards/StrictFormatReward/mean": 0.9416666507720948, "rewards/StrictFormatReward/std": 0.32878931760787966, "sampling/importance_sampling_ratio/max": 2.2147631645202637, "sampling/importance_sampling_ratio/mean": 0.2967061519622803, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.436277627944946, "sampling/sampling_logp_difference/mean": 0.02757907584309578, "step": 65, "step_time": 18.4663399906829 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1065.4, "completions/max_terminated_length": 922.2, "completions/mean_length": 740.2875244140625, "completions/mean_terminated_length": 736.9660522460938, "completions/min_length": 677.8, "completions/min_terminated_length": 677.8, "entropy": 0.69403355717659, "epoch": 0.010501050105010502, "frac_reward_zero_std": 0.0, "grad_norm": 0.609375, "learning_rate": 8.345323741007194e-07, "loss": -0.0133, "num_tokens": 7807277.0, "reward": 0.5900996208190918, "reward_std": 0.1928822249174118, "rewards/ADEnReward/mean": 0.020568987634032965, "rewards/ADEnReward/std": 0.04661810412071645, "rewards/CriticEdgeReward/mean": 0.4745306134223938, "rewards/CriticEdgeReward/std": 0.17837992906570435, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.2710542917251587, "sampling/importance_sampling_ratio/max": 2.8842154502868653, "sampling/importance_sampling_ratio/mean": 0.4043130338191986, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1025741577148436, "sampling/sampling_logp_difference/mean": 0.027092699706554414, "step": 70, "step_time": 19.328395317774266 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 809.6, "completions/max_terminated_length": 809.6, "completions/mean_length": 734.1458618164063, "completions/mean_terminated_length": 734.1458618164063, "completions/min_length": 657.8, "completions/min_terminated_length": 657.8, "entropy": 0.6823286473751068, "epoch": 0.011251125112511251, "frac_reward_zero_std": 0.0, "grad_norm": 1.234375, "learning_rate": 8.225419664268585e-07, "loss": -0.0095, "num_tokens": 8364416.0, "reward": 0.5976110935211182, "reward_std": 0.19164975583553315, "rewards/ADEnReward/mean": 0.01911538578569889, "rewards/ADEnReward/std": 0.046160650253295896, "rewards/CriticEdgeReward/mean": 0.4826623499393463, "rewards/CriticEdgeReward/std": 0.1823015034198761, "rewards/StrictFormatReward/mean": 0.9583333373069763, "rewards/StrictFormatReward/std": 0.21928777694702148, "sampling/importance_sampling_ratio/max": 2.258282995223999, "sampling/importance_sampling_ratio/mean": 0.38357329964637754, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.9673471927642823, "sampling/sampling_logp_difference/mean": 0.027145153656601907, "step": 75, "step_time": 17.740523260831832 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1089.0, "completions/max_terminated_length": 1089.0, "completions/mean_length": 742.108349609375, "completions/mean_terminated_length": 742.108349609375, "completions/min_length": 562.2, "completions/min_terminated_length": 562.2, "entropy": 0.6755726337432861, "epoch": 0.012001200120012, "frac_reward_zero_std": 0.0, "grad_norm": 1.0625, "learning_rate": 8.105515587529975e-07, "loss": -0.051, "num_tokens": 8923642.0, "reward": 0.6673060655593872, "reward_std": 0.2284113198518753, "rewards/ADEnReward/mean": 0.032044133823364976, "rewards/ADEnReward/std": 0.0727558545768261, "rewards/CriticEdgeReward/mean": 0.5410952270030975, "rewards/CriticEdgeReward/std": 0.20587096214294434, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.2538138389587403, "sampling/importance_sampling_ratio/mean": 0.3769398331642151, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8417075157165526, "sampling/sampling_logp_difference/mean": 0.027786290273070335, "step": 80, "step_time": 19.63715806733817 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 959.0, "completions/max_terminated_length": 959.0, "completions/mean_length": 734.083349609375, "completions/mean_terminated_length": 734.083349609375, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.6718262612819672, "epoch": 0.012751275127512752, "frac_reward_zero_std": 0.0, "grad_norm": 0.8828125, "learning_rate": 7.985611510791367e-07, "loss": -0.0065, "num_tokens": 9480574.0, "reward": 0.6821875452995301, "reward_std": 0.2173808127641678, "rewards/ADEnReward/mean": 0.029634485393762587, "rewards/ADEnReward/std": 0.0647291123867035, "rewards/CriticEdgeReward/mean": 0.5567197203636169, "rewards/CriticEdgeReward/std": 0.2098926305770874, "rewards/StrictFormatReward/mean": 0.9583333373069763, "rewards/StrictFormatReward/std": 0.21928777694702148, "sampling/importance_sampling_ratio/max": 2.5284235000610353, "sampling/importance_sampling_ratio/mean": 0.3952642858028412, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1834614992141725, "sampling/sampling_logp_difference/mean": 0.026568666100502014, "step": 85, "step_time": 18.634345239400865 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1059.6, "completions/max_terminated_length": 1059.6, "completions/mean_length": 735.3708374023438, "completions/mean_terminated_length": 735.3708374023438, "completions/min_length": 638.8, "completions/min_terminated_length": 638.8, "entropy": 0.6840767562389374, "epoch": 0.013501350135013501, "frac_reward_zero_std": 0.0, "grad_norm": 0.427734375, "learning_rate": 7.865707434052757e-07, "loss": 0.0256, "num_tokens": 10037783.0, "reward": 0.667415201663971, "reward_std": 0.22736435532569885, "rewards/ADEnReward/mean": 0.04481395985931158, "rewards/ADEnReward/std": 0.10050129070878029, "rewards/CriticEdgeReward/mean": 0.5292678833007812, "rewards/CriticEdgeReward/std": 0.19461615085601808, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.35183066725730894, "sampling/importance_sampling_ratio/max": 2.384801816940308, "sampling/importance_sampling_ratio/mean": 0.3732164800167084, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.599190521240234, "sampling/sampling_logp_difference/mean": 0.02623758539557457, "step": 90, "step_time": 19.2336613336578 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 956.0, "completions/max_terminated_length": 813.6, "completions/mean_length": 733.3250244140625, "completions/mean_terminated_length": 729.966943359375, "completions/min_length": 664.0, "completions/min_terminated_length": 664.0, "entropy": 0.6803362786769866, "epoch": 0.014251425142514252, "frac_reward_zero_std": 0.0, "grad_norm": 0.62890625, "learning_rate": 7.745803357314147e-07, "loss": 0.048, "num_tokens": 10594933.0, "reward": 0.6178993225097656, "reward_std": 0.1994838148355484, "rewards/ADEnReward/mean": 0.016964195016771554, "rewards/ADEnReward/std": 0.04836812801659107, "rewards/CriticEdgeReward/mean": 0.5076017618179322, "rewards/CriticEdgeReward/std": 0.1879289209842682, "rewards/StrictFormatReward/mean": 0.9333333492279052, "rewards/StrictFormatReward/std": 0.32310550212860106, "sampling/importance_sampling_ratio/max": 2.56809515953064, "sampling/importance_sampling_ratio/mean": 0.3793637275695801, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3464714527130126, "sampling/sampling_logp_difference/mean": 0.02770846411585808, "step": 95, "step_time": 18.659732691850515 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1042.8, "completions/max_terminated_length": 897.2, "completions/mean_length": 731.31669921875, "completions/mean_terminated_length": 727.953564453125, "completions/min_length": 553.2, "completions/min_terminated_length": 553.2, "entropy": 0.6775740444660187, "epoch": 0.015001500150015001, "frac_reward_zero_std": 0.0, "grad_norm": 0.98828125, "learning_rate": 7.62589928057554e-07, "loss": -0.0052, "num_tokens": 11150753.0, "reward": 0.6541489720344543, "reward_std": 0.23090167939662934, "rewards/ADEnReward/mean": 0.041887599229812625, "rewards/ADEnReward/std": 0.08931041695177555, "rewards/CriticEdgeReward/mean": 0.5197613358497619, "rewards/CriticEdgeReward/std": 0.18792039155960083, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.37487201690673827, "sampling/importance_sampling_ratio/max": 2.6180293560028076, "sampling/importance_sampling_ratio/mean": 0.3991284668445587, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.96435866355896, "sampling/sampling_logp_difference/mean": 0.02632814794778824, "step": 100, "step_time": 19.24204530082643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 957.2, "completions/max_terminated_length": 812.8, "completions/mean_length": 737.3041870117188, "completions/mean_terminated_length": 733.9729736328125, "completions/min_length": 462.6, "completions/min_terminated_length": 462.6, "entropy": 0.677629142999649, "epoch": 0.01575157515751575, "frac_reward_zero_std": 0.0, "grad_norm": 0.50390625, "learning_rate": 7.50599520383693e-07, "loss": -0.0524, "num_tokens": 11709082.0, "reward": 0.6337182760238648, "reward_std": 0.22624589800834655, "rewards/ADEnReward/mean": 0.01744948886334896, "rewards/ADEnReward/std": 0.05185253694653511, "rewards/CriticEdgeReward/mean": 0.5229354441165924, "rewards/CriticEdgeReward/std": 0.20465186536312102, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.35183066725730894, "sampling/importance_sampling_ratio/max": 2.4730069160461428, "sampling/importance_sampling_ratio/mean": 0.38834974765777586, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.500089979171753, "sampling/sampling_logp_difference/mean": 0.027956438809633256, "step": 105, "step_time": 18.564498315565288 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1067.0, "completions/max_terminated_length": 925.8, "completions/mean_length": 734.5375244140625, "completions/mean_terminated_length": 731.1711059570313, "completions/min_length": 653.4, "completions/min_terminated_length": 653.4, "entropy": 0.6756197333335876, "epoch": 0.0165016501650165, "frac_reward_zero_std": 0.0, "grad_norm": 0.5390625, "learning_rate": 7.386091127098321e-07, "loss": -0.0704, "num_tokens": 12266075.0, "reward": 0.6165385961532592, "reward_std": 0.22728370726108552, "rewards/ADEnReward/mean": 0.02051542168483138, "rewards/ADEnReward/std": 0.06193199865520001, "rewards/CriticEdgeReward/mean": 0.5035231411457062, "rewards/CriticEdgeReward/std": 0.19765084683895112, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.3342098295688629, "sampling/importance_sampling_ratio/max": 2.680658149719238, "sampling/importance_sampling_ratio/mean": 0.35913369059562683, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.405208158493042, "sampling/sampling_logp_difference/mean": 0.02704847790300846, "step": 110, "step_time": 19.224649833049625 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1057.4, "completions/max_terminated_length": 1057.4, "completions/mean_length": 742.3750122070312, "completions/mean_terminated_length": 742.3750122070312, "completions/min_length": 660.4, "completions/min_terminated_length": 660.4, "entropy": 0.6867500066757202, "epoch": 0.017251725172517253, "frac_reward_zero_std": 0.0, "grad_norm": 0.57421875, "learning_rate": 7.266187050359712e-07, "loss": -0.0143, "num_tokens": 12825397.0, "reward": 0.6780861854553223, "reward_std": 0.2309806227684021, "rewards/ADEnReward/mean": 0.0395726390182972, "rewards/ADEnReward/std": 0.08671161495149135, "rewards/CriticEdgeReward/mean": 0.545180207490921, "rewards/CriticEdgeReward/std": 0.20045872628688813, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3577991783618927, "sampling/importance_sampling_ratio/max": 2.3207207202911375, "sampling/importance_sampling_ratio/mean": 0.37505191564559937, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.2707075595855715, "sampling/sampling_logp_difference/mean": 0.02805633805692196, "step": 115, "step_time": 19.346580943930896 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 920.2, "completions/max_terminated_length": 920.2, "completions/mean_length": 729.8083374023438, "completions/mean_terminated_length": 729.8083374023438, "completions/min_length": 589.8, "completions/min_terminated_length": 589.8, "entropy": 0.6623412787914276, "epoch": 0.018001800180018002, "frac_reward_zero_std": 0.0, "grad_norm": 0.30859375, "learning_rate": 7.146282973621102e-07, "loss": -0.0363, "num_tokens": 13381543.0, "reward": 0.6440199613571167, "reward_std": 0.21797512769699096, "rewards/ADEnReward/mean": 0.026557114161550998, "rewards/ADEnReward/std": 0.06657822728157044, "rewards/CriticEdgeReward/mean": 0.5241294622421264, "rewards/CriticEdgeReward/std": 0.19584953784942627, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.30797049403190613, "sampling/importance_sampling_ratio/max": 2.2972915887832643, "sampling/importance_sampling_ratio/mean": 0.3180178701877594, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.329182481765747, "sampling/sampling_logp_difference/mean": 0.027167493104934694, "step": 120, "step_time": 18.50410401271656 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 804.2, "completions/max_terminated_length": 804.2, "completions/mean_length": 733.6416748046875, "completions/mean_terminated_length": 733.6416748046875, "completions/min_length": 665.2, "completions/min_terminated_length": 665.2, "entropy": 0.6787201821804046, "epoch": 0.01875187518751875, "frac_reward_zero_std": 0.0, "grad_norm": 0.515625, "learning_rate": 7.026378896882493e-07, "loss": 0.0414, "num_tokens": 13938513.0, "reward": 0.6720813035964965, "reward_std": 0.21677926778793336, "rewards/ADEnReward/mean": 0.023206131346523762, "rewards/ADEnReward/std": 0.06639735922217369, "rewards/CriticEdgeReward/mean": 0.5538751184940338, "rewards/CriticEdgeReward/std": 0.19620085060596465, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.27702280282974245, "sampling/importance_sampling_ratio/max": 2.243584942817688, "sampling/importance_sampling_ratio/mean": 0.3404032289981842, "sampling/importance_sampling_ratio/min": 0.0001499661710113287, "sampling/sampling_logp_difference/max": 3.585259485244751, "sampling/sampling_logp_difference/mean": 0.02763916291296482, "step": 125, "step_time": 17.699040143657477 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 941.4, "completions/max_terminated_length": 941.4, "completions/mean_length": 734.1250122070312, "completions/mean_terminated_length": 734.1250122070312, "completions/min_length": 559.4, "completions/min_terminated_length": 559.4, "entropy": 0.677570229768753, "epoch": 0.0195019501950195, "frac_reward_zero_std": 0.0, "grad_norm": 0.55859375, "learning_rate": 6.906474820143885e-07, "loss": -0.0276, "num_tokens": 14496239.0, "reward": 0.6338245630264282, "reward_std": 0.2223297268152237, "rewards/ADEnReward/mean": 0.02823296405840665, "rewards/ADEnReward/std": 0.06825487604364752, "rewards/CriticEdgeReward/mean": 0.5147582352161407, "rewards/CriticEdgeReward/std": 0.19871617555618287, "rewards/StrictFormatReward/mean": 0.9083333253860474, "rewards/StrictFormatReward/std": 0.3651575207710266, "sampling/importance_sampling_ratio/max": 2.270122742652893, "sampling/importance_sampling_ratio/mean": 0.3096809387207031, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.943360328674316, "sampling/sampling_logp_difference/mean": 0.027701665833592414, "step": 130, "step_time": 18.597071247920393 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 919.6, "completions/max_terminated_length": 919.6, "completions/mean_length": 732.5916870117187, "completions/mean_terminated_length": 732.5916870117187, "completions/min_length": 660.4, "completions/min_terminated_length": 660.4, "entropy": 0.6311272621154785, "epoch": 0.02025202520252025, "frac_reward_zero_std": 0.0, "grad_norm": 0.59375, "learning_rate": 6.786570743405276e-07, "loss": -0.0432, "num_tokens": 15053133.0, "reward": 0.6482689142227173, "reward_std": 0.2235964745283127, "rewards/ADEnReward/mean": 0.03200184367597103, "rewards/ADEnReward/std": 0.07811516597867012, "rewards/CriticEdgeReward/mean": 0.5229337155818939, "rewards/CriticEdgeReward/std": 0.2021573781967163, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.7773334980010986, "sampling/importance_sampling_ratio/mean": 0.35025218725204466, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.60329647064209, "sampling/sampling_logp_difference/mean": 0.02632040120661259, "step": 135, "step_time": 18.460146799404175 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1090.6, "completions/max_terminated_length": 803.8, "completions/mean_length": 734.2333374023438, "completions/mean_terminated_length": 727.5319946289062, "completions/min_length": 563.8, "completions/min_terminated_length": 563.8, "entropy": 0.6421706557273865, "epoch": 0.021002100210021003, "frac_reward_zero_std": 0.0, "grad_norm": 0.408203125, "learning_rate": 6.666666666666666e-07, "loss": -0.023, "num_tokens": 15610629.0, "reward": 0.6999334573745728, "reward_std": 0.2522183507680893, "rewards/ADEnReward/mean": 0.04455822389572859, "rewards/ADEnReward/std": 0.09245736598968506, "rewards/CriticEdgeReward/mean": 0.5628751993179322, "rewards/CriticEdgeReward/std": 0.21973196566104888, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.3250433325767517, "sampling/importance_sampling_ratio/max": 2.1399518966674806, "sampling/importance_sampling_ratio/mean": 0.31972921192646026, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.696309661865234, "sampling/sampling_logp_difference/mean": 0.02722916007041931, "step": 140, "step_time": 19.35403058277443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 880.6, "completions/max_terminated_length": 880.6, "completions/mean_length": 734.5916748046875, "completions/mean_terminated_length": 734.5916748046875, "completions/min_length": 662.6, "completions/min_terminated_length": 662.6, "entropy": 0.66073579788208, "epoch": 0.021752175217521753, "frac_reward_zero_std": 0.0, "grad_norm": 0.98828125, "learning_rate": 6.546762589928057e-07, "loss": -0.0672, "num_tokens": 16167955.0, "reward": 0.6939844369888306, "reward_std": 0.23545753657817842, "rewards/ADEnReward/mean": 0.03596792742609978, "rewards/ADEnReward/std": 0.08121550381183625, "rewards/CriticEdgeReward/mean": 0.5638498067855835, "rewards/CriticEdgeReward/std": 0.20780062079429626, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.30006415247917173, "sampling/importance_sampling_ratio/max": 2.2731374740600585, "sampling/importance_sampling_ratio/mean": 0.36867680549621584, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.253248405456543, "sampling/sampling_logp_difference/mean": 0.026698021590709685, "step": 145, "step_time": 18.200165270548315 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 804.2, "completions/max_terminated_length": 804.2, "completions/mean_length": 729.3791870117187, "completions/mean_terminated_length": 729.3791870117187, "completions/min_length": 550.4, "completions/min_terminated_length": 550.4, "entropy": 0.6892345249652863, "epoch": 0.022502250225022502, "frac_reward_zero_std": 0.0, "grad_norm": 0.67578125, "learning_rate": 6.426858513189448e-07, "loss": -0.0016, "num_tokens": 16723950.0, "reward": 0.6240308880805969, "reward_std": 0.2186569392681122, "rewards/ADEnReward/mean": 0.015108689107000828, "rewards/ADEnReward/std": 0.0516890412196517, "rewards/CriticEdgeReward/mean": 0.5180888473987579, "rewards/CriticEdgeReward/std": 0.19708221554756164, "rewards/StrictFormatReward/mean": 0.9083333492279053, "rewards/StrictFormatReward/std": 0.42095471620559693, "sampling/importance_sampling_ratio/max": 2.2781413793563843, "sampling/importance_sampling_ratio/mean": 0.3688820660114288, "sampling/importance_sampling_ratio/min": 2.7399107693781843e-07, "sampling/sampling_logp_difference/max": 6.100681638717651, "sampling/sampling_logp_difference/mean": 0.028175700455904007, "step": 150, "step_time": 17.736496021598576 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1171.8, "completions/max_terminated_length": 1171.8, "completions/mean_length": 740.895849609375, "completions/mean_terminated_length": 740.895849609375, "completions/min_length": 547.4, "completions/min_terminated_length": 547.4, "entropy": 0.6601650178432464, "epoch": 0.02325232523252325, "frac_reward_zero_std": 0.0, "grad_norm": 1.3046875, "learning_rate": 6.306954436450838e-07, "loss": -0.0441, "num_tokens": 17282693.0, "reward": 0.6705703139305115, "reward_std": 0.23699760138988496, "rewards/ADEnReward/mean": 0.03684302102774382, "rewards/ADEnReward/std": 0.07903052866458893, "rewards/CriticEdgeReward/mean": 0.5420606195926666, "rewards/CriticEdgeReward/std": 0.19771154820919037, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.39194485545158386, "sampling/importance_sampling_ratio/max": 2.78102593421936, "sampling/importance_sampling_ratio/mean": 0.4051141023635864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.800427055358886, "sampling/sampling_logp_difference/mean": 0.027090968936681746, "step": 155, "step_time": 19.98870733361691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1074.6, "completions/max_terminated_length": 1074.6, "completions/mean_length": 728.8416870117187, "completions/mean_terminated_length": 728.8416870117187, "completions/min_length": 670.2, "completions/min_terminated_length": 670.2, "entropy": 0.6536854803562164, "epoch": 0.024002400240024, "frac_reward_zero_std": 0.0, "grad_norm": 0.455078125, "learning_rate": 6.187050359712231e-07, "loss": -0.0618, "num_tokens": 17838271.0, "reward": 0.6057126641273498, "reward_std": 0.19732293784618377, "rewards/ADEnReward/mean": 0.016158161405473947, "rewards/ADEnReward/std": 0.03743497058749199, "rewards/CriticEdgeReward/mean": 0.49872114658355715, "rewards/CriticEdgeReward/std": 0.1852245330810547, "rewards/StrictFormatReward/mean": 0.9083333253860474, "rewards/StrictFormatReward/std": 0.4058197200298309, "sampling/importance_sampling_ratio/max": 2.264492416381836, "sampling/importance_sampling_ratio/mean": 0.39292792677879335, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.323010778427124, "sampling/sampling_logp_difference/mean": 0.026840243861079215, "step": 160, "step_time": 19.39901271658018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 727.6625366210938, "completions/mean_terminated_length": 727.6625366210938, "completions/min_length": 563.8, "completions/min_terminated_length": 563.8, "entropy": 0.664479774236679, "epoch": 0.024752475247524754, "frac_reward_zero_std": 0.0, "grad_norm": 0.8125, "learning_rate": 6.067146282973621e-07, "loss": -0.0233, "num_tokens": 18393310.0, "reward": 0.6312724232673645, "reward_std": 0.20918006598949432, "rewards/ADEnReward/mean": 0.03701213132590055, "rewards/ADEnReward/std": 0.08919502422213554, "rewards/CriticEdgeReward/mean": 0.4975935935974121, "rewards/CriticEdgeReward/std": 0.18937192261219024, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.3564636945724486, "sampling/importance_sampling_ratio/mean": 0.4049984455108643, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5581103563308716, "sampling/sampling_logp_difference/mean": 0.026597682759165762, "step": 165, "step_time": 17.78500230628997 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 805.2, "completions/max_terminated_length": 805.2, "completions/mean_length": 732.5666748046875, "completions/mean_terminated_length": 732.5666748046875, "completions/min_length": 674.2, "completions/min_terminated_length": 674.2, "entropy": 0.6670410990715027, "epoch": 0.025502550255025503, "frac_reward_zero_std": 0.0, "grad_norm": 0.4921875, "learning_rate": 5.947242206235011e-07, "loss": -0.0536, "num_tokens": 18949974.0, "reward": 0.64571533203125, "reward_std": 0.2203593909740448, "rewards/ADEnReward/mean": 0.028900963254272938, "rewards/ADEnReward/std": 0.06595891937613488, "rewards/CriticEdgeReward/mean": 0.5209810256958007, "rewards/CriticEdgeReward/std": 0.19441207349300385, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.21331926584243774, "sampling/importance_sampling_ratio/max": 2.4715036153793335, "sampling/importance_sampling_ratio/mean": 0.37501450777053835, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.4874019384384156, "sampling/sampling_logp_difference/mean": 0.026845839247107504, "step": 170, "step_time": 17.687379773985594 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 811.2, "completions/max_terminated_length": 811.2, "completions/mean_length": 731.2791870117187, "completions/mean_terminated_length": 731.2791870117187, "completions/min_length": 596.2, "completions/min_terminated_length": 596.2, "entropy": 0.6984724462032318, "epoch": 0.026252625262526252, "frac_reward_zero_std": 0.0, "grad_norm": 1.0, "learning_rate": 5.827338129496403e-07, "loss": -0.0679, "num_tokens": 19506473.0, "reward": 0.6512439846992493, "reward_std": 0.22553048133850098, "rewards/ADEnReward/mean": 0.02240049056708813, "rewards/ADEnReward/std": 0.06565947569906712, "rewards/CriticEdgeReward/mean": 0.5313434898853302, "rewards/CriticEdgeReward/std": 0.2045954704284668, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.4593673229217528, "sampling/importance_sampling_ratio/mean": 0.34647259712219236, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.540093183517456, "sampling/sampling_logp_difference/mean": 0.02706814929842949, "step": 175, "step_time": 17.735902212653308 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1059.0, "completions/max_terminated_length": 1059.0, "completions/mean_length": 738.25419921875, "completions/mean_terminated_length": 738.25419921875, "completions/min_length": 567.4, "completions/min_terminated_length": 567.4, "entropy": 0.6531117916107178, "epoch": 0.027002700270027002, "frac_reward_zero_std": 0.0, "grad_norm": 1.0546875, "learning_rate": 5.707434052757793e-07, "loss": -0.0459, "num_tokens": 20064726.0, "reward": 0.671851909160614, "reward_std": 0.2211885303258896, "rewards/ADEnReward/mean": 0.03779828287661076, "rewards/ADEnReward/std": 0.08073382154107094, "rewards/CriticEdgeReward/mean": 0.5423869371414185, "rewards/CriticEdgeReward/std": 0.19303695857524872, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.39194485545158386, "sampling/importance_sampling_ratio/max": 2.3665098190307616, "sampling/importance_sampling_ratio/mean": 0.4010015606880188, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.2859954833984375, "sampling/sampling_logp_difference/mean": 0.0267644289880991, "step": 180, "step_time": 19.318453999981283 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 792.8, "completions/max_terminated_length": 792.8, "completions/mean_length": 733.2541748046875, "completions/mean_terminated_length": 733.2541748046875, "completions/min_length": 659.6, "completions/min_terminated_length": 659.6, "entropy": 0.6468486309051513, "epoch": 0.02775277527752775, "frac_reward_zero_std": 0.0, "grad_norm": 0.42578125, "learning_rate": 5.587529976019184e-07, "loss": -0.0417, "num_tokens": 20621859.0, "reward": 0.683180284500122, "reward_std": 0.22336726486682892, "rewards/ADEnReward/mean": 0.026478290744125842, "rewards/ADEnReward/std": 0.061912165582180025, "rewards/CriticEdgeReward/mean": 0.5608686208724976, "rewards/CriticEdgeReward/std": 0.20488823652267457, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.2539814531803131, "sampling/importance_sampling_ratio/max": 2.1877712488174437, "sampling/importance_sampling_ratio/mean": 0.3163671612739563, "sampling/importance_sampling_ratio/min": 2.460074611008167e-05, "sampling/sampling_logp_difference/max": 3.400425410270691, "sampling/sampling_logp_difference/mean": 0.027360597625374794, "step": 185, "step_time": 17.668436461221425 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1113.6, "completions/max_terminated_length": 970.2, "completions/mean_length": 742.5458618164063, "completions/mean_terminated_length": 739.184326171875, "completions/min_length": 430.8, "completions/min_terminated_length": 430.8, "entropy": 0.6834874272346496, "epoch": 0.028502850285028504, "frac_reward_zero_std": 0.0, "grad_norm": 0.65625, "learning_rate": 5.467625899280576e-07, "loss": -0.0541, "num_tokens": 21181254.0, "reward": 0.6345921754837036, "reward_std": 0.2239381343126297, "rewards/ADEnReward/mean": 0.02896077521145344, "rewards/ADEnReward/std": 0.07604779824614524, "rewards/CriticEdgeReward/mean": 0.5131313860416412, "rewards/CriticEdgeReward/std": 0.20311089158058165, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.36890350580215453, "sampling/importance_sampling_ratio/max": 2.373605179786682, "sampling/importance_sampling_ratio/mean": 0.31607483625411986, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.902349233627319, "sampling/sampling_logp_difference/mean": 0.027898192033171655, "step": 190, "step_time": 19.545805886760355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 797.6, "completions/max_terminated_length": 797.6, "completions/mean_length": 728.5250122070313, "completions/mean_terminated_length": 728.5250122070313, "completions/min_length": 652.8, "completions/min_terminated_length": 652.8, "entropy": 0.6718811094760895, "epoch": 0.029252925292529253, "frac_reward_zero_std": 0.0, "grad_norm": 0.43359375, "learning_rate": 5.347721822541966e-07, "loss": -0.0201, "num_tokens": 21736788.0, "reward": 0.6331563830375672, "reward_std": 0.21693200170993804, "rewards/ADEnReward/mean": 0.04175339005887509, "rewards/ADEnReward/std": 0.10068345740437508, "rewards/CriticEdgeReward/mean": 0.4972362995147705, "rewards/CriticEdgeReward/std": 0.18756451904773713, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.3347578287124634, "sampling/importance_sampling_ratio/max": 2.160827088356018, "sampling/importance_sampling_ratio/mean": 0.351710706949234, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.30830945968628, "sampling/sampling_logp_difference/mean": 0.02647450603544712, "step": 195, "step_time": 17.642177342809738 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1081.6, "completions/max_terminated_length": 935.6, "completions/mean_length": 737.7250244140625, "completions/mean_terminated_length": 734.3415893554687, "completions/min_length": 558.4, "completions/min_terminated_length": 558.4, "entropy": 0.6970606863498687, "epoch": 0.030003000300030003, "frac_reward_zero_std": 0.0, "grad_norm": 0.7265625, "learning_rate": 5.227817745803357e-07, "loss": -0.0635, "num_tokens": 22294818.0, "reward": 0.6249044418334961, "reward_std": 0.22049946188926697, "rewards/ADEnReward/mean": 0.01257529379799962, "rewards/ADEnReward/std": 0.03925408348441124, "rewards/CriticEdgeReward/mean": 0.5198291122913361, "rewards/CriticEdgeReward/std": 0.19913202822208403, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.37487201690673827, "sampling/importance_sampling_ratio/max": 2.8745041847229005, "sampling/importance_sampling_ratio/mean": 0.4910339951515198, "sampling/importance_sampling_ratio/min": 0.0004650283139199018, "sampling/sampling_logp_difference/max": 4.296806478500367, "sampling/sampling_logp_difference/mean": 0.027369559928774833, "step": 200, "step_time": 19.332767476607113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 813.4, "completions/max_terminated_length": 813.4, "completions/mean_length": 733.4625244140625, "completions/mean_terminated_length": 733.4625244140625, "completions/min_length": 677.0, "completions/min_terminated_length": 677.0, "entropy": 0.6551679193973541, "epoch": 0.030753075307530752, "frac_reward_zero_std": 0.0, "grad_norm": 0.46875, "learning_rate": 5.107913669064748e-07, "loss": -0.0754, "num_tokens": 22851809.0, "reward": 0.6802987098693848, "reward_std": 0.2658484667539597, "rewards/ADEnReward/mean": 0.03954121433198452, "rewards/ADEnReward/std": 0.0957181766629219, "rewards/CriticEdgeReward/mean": 0.5424241185188293, "rewards/CriticEdgeReward/std": 0.21895764470100404, "rewards/StrictFormatReward/mean": 0.9833333253860473, "rewards/StrictFormatReward/std": 0.1154700517654419, "sampling/importance_sampling_ratio/max": 2.3209081411361696, "sampling/importance_sampling_ratio/mean": 0.31685432195663454, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.857770776748657, "sampling/sampling_logp_difference/mean": 0.026842932403087615, "step": 205, "step_time": 18.285551036614926 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.4, "completions/max_terminated_length": 802.4, "completions/mean_length": 726.2583374023437, "completions/mean_terminated_length": 726.2583374023437, "completions/min_length": 626.4, "completions/min_terminated_length": 626.4, "entropy": 0.6476575255393981, "epoch": 0.0315031503150315, "frac_reward_zero_std": 0.0, "grad_norm": 0.59375, "learning_rate": 4.988009592326139e-07, "loss": -0.0105, "num_tokens": 23407199.0, "reward": 0.647519564628601, "reward_std": 0.2094862699508667, "rewards/ADEnReward/mean": 0.015838828310370444, "rewards/ADEnReward/std": 0.03897444941103458, "rewards/CriticEdgeReward/mean": 0.5366807103157043, "rewards/CriticEdgeReward/std": 0.19412737488746643, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.18121873140335082, "sampling/importance_sampling_ratio/max": 2.376599931716919, "sampling/importance_sampling_ratio/mean": 0.4298800349235535, "sampling/importance_sampling_ratio/min": 1.0286225005984306e-05, "sampling/sampling_logp_difference/max": 5.127938175201416, "sampling/sampling_logp_difference/mean": 0.02668023928999901, "step": 210, "step_time": 17.68627334833145 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1081.8, "completions/max_terminated_length": 1062.4, "completions/mean_length": 743.6125244140625, "completions/mean_terminated_length": 740.2939819335937, "completions/min_length": 637.6, "completions/min_terminated_length": 637.6, "entropy": 0.670571756362915, "epoch": 0.032253225322532254, "frac_reward_zero_std": 0.0, "grad_norm": 0.48046875, "learning_rate": 4.86810551558753e-07, "loss": -0.0164, "num_tokens": 23966914.0, "reward": 0.6632338404655457, "reward_std": 0.22702765762805938, "rewards/ADEnReward/mean": 0.03454024754464626, "rewards/ADEnReward/std": 0.08139340579509735, "rewards/CriticEdgeReward/mean": 0.5395268857479095, "rewards/CriticEdgeReward/std": 0.2030743181705475, "rewards/StrictFormatReward/mean": 0.8916666746139527, "rewards/StrictFormatReward/std": 0.45510039329528806, "sampling/importance_sampling_ratio/max": 2.3546474456787108, "sampling/importance_sampling_ratio/mean": 0.3652418226003647, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.994574356079101, "sampling/sampling_logp_difference/mean": 0.027023188397288324, "step": 215, "step_time": 19.507186018303038 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1085.4, "completions/max_terminated_length": 937.4, "completions/mean_length": 739.5166870117188, "completions/mean_terminated_length": 736.1462036132813, "completions/min_length": 658.8, "completions/min_terminated_length": 658.8, "entropy": 0.6571417331695557, "epoch": 0.033003300330033, "frac_reward_zero_std": 0.0, "grad_norm": 0.63671875, "learning_rate": 4.7482014388489204e-07, "loss": -0.0841, "num_tokens": 24525742.0, "reward": 0.6571517825126648, "reward_std": 0.223968243598938, "rewards/ADEnReward/mean": 0.0204413834027946, "rewards/ADEnReward/std": 0.056050950288772584, "rewards/CriticEdgeReward/mean": 0.5425436973571778, "rewards/CriticEdgeReward/std": 0.19919070601463318, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.445080280303955, "sampling/importance_sampling_ratio/mean": 0.36826732754707336, "sampling/importance_sampling_ratio/min": 3.5697961720870807e-06, "sampling/sampling_logp_difference/max": 4.370722007751465, "sampling/sampling_logp_difference/mean": 0.026936203613877295, "step": 220, "step_time": 19.575943036191166 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 870.2, "completions/max_terminated_length": 870.2, "completions/mean_length": 728.4125122070312, "completions/mean_terminated_length": 728.4125122070312, "completions/min_length": 656.8, "completions/min_terminated_length": 656.8, "entropy": 0.6509746253490448, "epoch": 0.03375337533753375, "frac_reward_zero_std": 0.0, "grad_norm": 0.9296875, "learning_rate": 4.628297362110312e-07, "loss": -0.0833, "num_tokens": 25081457.0, "reward": 0.6875805377960205, "reward_std": 0.24434725642204286, "rewards/ADEnReward/mean": 0.03638526238501072, "rewards/ADEnReward/std": 0.08670183569192887, "rewards/CriticEdgeReward/mean": 0.5578618943691254, "rewards/CriticEdgeReward/std": 0.20448245704174042, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3577991783618927, "sampling/importance_sampling_ratio/max": 2.328741121292114, "sampling/importance_sampling_ratio/mean": 0.3569604575634003, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.641915273666382, "sampling/sampling_logp_difference/mean": 0.026573105528950692, "step": 225, "step_time": 18.2410368247889 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1101.6, "completions/max_terminated_length": 810.8, "completions/mean_length": 736.10419921875, "completions/mean_terminated_length": 729.3391845703125, "completions/min_length": 657.8, "completions/min_terminated_length": 657.8, "entropy": 0.6685130417346954, "epoch": 0.034503450345034506, "frac_reward_zero_std": 0.0, "grad_norm": 0.458984375, "learning_rate": 4.508393285371702e-07, "loss": 0.0183, "num_tokens": 25639354.0, "reward": 0.6798216700553894, "reward_std": 0.2329070210456848, "rewards/ADEnReward/mean": 0.02320026506204158, "rewards/ADEnReward/std": 0.05984626142308116, "rewards/CriticEdgeReward/mean": 0.5657880663871765, "rewards/CriticEdgeReward/std": 0.21779701113700867, "rewards/StrictFormatReward/mean": 0.9083333373069763, "rewards/StrictFormatReward/std": 0.4149862051010132, "sampling/importance_sampling_ratio/max": 2.4524559497833254, "sampling/importance_sampling_ratio/mean": 0.37915992736816406, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1569953203201293, "sampling/sampling_logp_difference/mean": 0.027354817464947702, "step": 230, "step_time": 19.698453634697945 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 947.2, "completions/max_terminated_length": 947.2, "completions/mean_length": 731.12919921875, "completions/mean_terminated_length": 731.12919921875, "completions/min_length": 558.8, "completions/min_terminated_length": 558.8, "entropy": 0.6658551037311554, "epoch": 0.03525352535253525, "frac_reward_zero_std": 0.0, "grad_norm": 0.412109375, "learning_rate": 4.388489208633093e-07, "loss": 0.0387, "num_tokens": 26195849.0, "reward": 0.660690426826477, "reward_std": 0.2438534140586853, "rewards/ADEnReward/mean": 0.024356142431497575, "rewards/ADEnReward/std": 0.06747710704803467, "rewards/CriticEdgeReward/mean": 0.5421675562858581, "rewards/CriticEdgeReward/std": 0.21383192241191865, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.260256600379944, "sampling/importance_sampling_ratio/mean": 0.3355040669441223, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.679680252075196, "sampling/sampling_logp_difference/mean": 0.027135537937283516, "step": 235, "step_time": 18.579185033869 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 811.0, "completions/max_terminated_length": 811.0, "completions/mean_length": 725.745849609375, "completions/mean_terminated_length": 725.745849609375, "completions/min_length": 660.2, "completions/min_terminated_length": 660.2, "entropy": 0.6525954186916352, "epoch": 0.036003600360036005, "frac_reward_zero_std": 0.0, "grad_norm": 0.6796875, "learning_rate": 4.2685851318944845e-07, "loss": -0.0861, "num_tokens": 26751068.0, "reward": 0.6906341195106507, "reward_std": 0.22764017879962922, "rewards/ADEnReward/mean": 0.037389728240668774, "rewards/ADEnReward/std": 0.0835967343300581, "rewards/CriticEdgeReward/mean": 0.5574110388755799, "rewards/CriticEdgeReward/std": 0.21200692057609558, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.21331926584243774, "sampling/importance_sampling_ratio/max": 2.383519434928894, "sampling/importance_sampling_ratio/mean": 0.38454734086990355, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.34567346572876, "sampling/sampling_logp_difference/mean": 0.027424708008766174, "step": 240, "step_time": 17.736516166571526 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1082.2, "completions/max_terminated_length": 1082.2, "completions/mean_length": 739.00419921875, "completions/mean_terminated_length": 739.00419921875, "completions/min_length": 669.6, "completions/min_terminated_length": 669.6, "entropy": 0.6817317605018616, "epoch": 0.03675367536753675, "frac_reward_zero_std": 0.0, "grad_norm": 0.46875, "learning_rate": 4.148681055155875e-07, "loss": 0.025, "num_tokens": 27309213.0, "reward": 0.6386038899421692, "reward_std": 0.21788020431995392, "rewards/ADEnReward/mean": 0.026447905600070952, "rewards/ADEnReward/std": 0.07397268936038018, "rewards/CriticEdgeReward/mean": 0.5171559512615204, "rewards/CriticEdgeReward/std": 0.18868504762649535, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.31171647906303407, "sampling/importance_sampling_ratio/max": 2.654625940322876, "sampling/importance_sampling_ratio/mean": 0.3686636805534363, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5372043371200563, "sampling/sampling_logp_difference/mean": 0.026847490295767783, "step": 245, "step_time": 19.36604656809941 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1232.0, "completions/max_terminated_length": 1232.0, "completions/mean_length": 732.945849609375, "completions/mean_terminated_length": 732.945849609375, "completions/min_length": 466.4, "completions/min_terminated_length": 466.4, "entropy": 0.6510292887687683, "epoch": 0.0375037503750375, "frac_reward_zero_std": 0.0, "grad_norm": 0.7109375, "learning_rate": 4.028776978417266e-07, "loss": 0.0165, "num_tokens": 27866048.0, "reward": 0.6444871425628662, "reward_std": 0.22737836241722106, "rewards/ADEnReward/mean": 0.019454053416848183, "rewards/ADEnReward/std": 0.05568660330027342, "rewards/CriticEdgeReward/mean": 0.5358664035797119, "rewards/CriticEdgeReward/std": 0.20160456001758575, "rewards/StrictFormatReward/mean": 0.8916666746139527, "rewards/StrictFormatReward/std": 0.45510039329528806, "sampling/importance_sampling_ratio/max": 2.237666130065918, "sampling/importance_sampling_ratio/mean": 0.35609259605407717, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.025123119354248, "sampling/sampling_logp_difference/mean": 0.027364023402333258, "step": 250, "step_time": 20.30714185880497 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 805.4, "completions/max_terminated_length": 805.4, "completions/mean_length": 731.31669921875, "completions/mean_terminated_length": 731.31669921875, "completions/min_length": 657.4, "completions/min_terminated_length": 657.4, "entropy": 0.6548093140125275, "epoch": 0.038253825382538256, "frac_reward_zero_std": 0.0, "grad_norm": 0.90625, "learning_rate": 3.908872901678657e-07, "loss": -0.008, "num_tokens": 28422684.0, "reward": 0.6841266512870788, "reward_std": 0.21991626918315887, "rewards/ADEnReward/mean": 0.02979475948959589, "rewards/ADEnReward/std": 0.07170313075184823, "rewards/CriticEdgeReward/mean": 0.56099853515625, "rewards/CriticEdgeReward/std": 0.1967328429222107, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.4599730730056764, "sampling/importance_sampling_ratio/mean": 0.34424332082271575, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.761904239654541, "sampling/sampling_logp_difference/mean": 0.02734643779695034, "step": 255, "step_time": 17.74388456940651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 830.0, "completions/max_terminated_length": 830.0, "completions/mean_length": 728.6125244140625, "completions/mean_terminated_length": 728.6125244140625, "completions/min_length": 657.6, "completions/min_terminated_length": 657.6, "entropy": 0.6534522712230683, "epoch": 0.039003900390039, "frac_reward_zero_std": 0.0, "grad_norm": 0.37109375, "learning_rate": 3.7889688249400476e-07, "loss": -0.0399, "num_tokens": 28978447.0, "reward": 0.6671910166740418, "reward_std": 0.22425581216812135, "rewards/ADEnReward/mean": 0.029241904988884924, "rewards/ADEnReward/std": 0.05915460288524628, "rewards/CriticEdgeReward/mean": 0.5412824034690857, "rewards/CriticEdgeReward/std": 0.19969309270381927, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.1555842399597168, "sampling/importance_sampling_ratio/max": 2.277798819541931, "sampling/importance_sampling_ratio/mean": 0.3665468215942383, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.234524989128113, "sampling/sampling_logp_difference/mean": 0.02658945694565773, "step": 260, "step_time": 17.841105895582587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 853.6, "completions/max_terminated_length": 853.6, "completions/mean_length": 729.8208618164062, "completions/mean_terminated_length": 729.8208618164062, "completions/min_length": 666.4, "completions/min_terminated_length": 666.4, "entropy": 0.672957056760788, "epoch": 0.039753975397539755, "frac_reward_zero_std": 0.0, "grad_norm": 0.70703125, "learning_rate": 3.6690647482014385e-07, "loss": -0.0167, "num_tokens": 29534420.0, "reward": 0.680908715724945, "reward_std": 0.22301371097564698, "rewards/ADEnReward/mean": 0.03240678794682026, "rewards/ADEnReward/std": 0.08294596597552299, "rewards/CriticEdgeReward/mean": 0.5526685595512391, "rewards/CriticEdgeReward/std": 0.20977894067764283, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.21331926584243774, "sampling/importance_sampling_ratio/max": 2.550454783439636, "sampling/importance_sampling_ratio/mean": 0.43814687728881835, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.568864226341248, "sampling/sampling_logp_difference/mean": 0.026665230095386506, "step": 265, "step_time": 18.25555696412921 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1056.2, "completions/max_terminated_length": 1056.2, "completions/mean_length": 733.5500244140625, "completions/mean_terminated_length": 733.5500244140625, "completions/min_length": 537.8, "completions/min_terminated_length": 537.8, "entropy": 0.660488486289978, "epoch": 0.0405040504050405, "frac_reward_zero_std": 0.0, "grad_norm": 0.9296875, "learning_rate": 3.54916067146283e-07, "loss": -0.0363, "num_tokens": 30091352.0, "reward": 0.6483618021011353, "reward_std": 0.2317020356655121, "rewards/ADEnReward/mean": 0.0339603565633297, "rewards/ADEnReward/std": 0.07790775336325169, "rewards/CriticEdgeReward/mean": 0.5202347218990326, "rewards/CriticEdgeReward/std": 0.19626265168190002, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.25343345403671264, "sampling/importance_sampling_ratio/max": 2.423857307434082, "sampling/importance_sampling_ratio/mean": 0.3932728707790375, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.533544635772705, "sampling/sampling_logp_difference/mean": 0.02748345360159874, "step": 270, "step_time": 19.24101545671001 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 815.4, "completions/max_terminated_length": 815.4, "completions/mean_length": 736.5666870117187, "completions/mean_terminated_length": 736.5666870117187, "completions/min_length": 658.4, "completions/min_terminated_length": 658.4, "entropy": 0.6971414148807525, "epoch": 0.041254125412541254, "frac_reward_zero_std": 0.0, "grad_norm": 0.6953125, "learning_rate": 3.4292565947242203e-07, "loss": 0.008, "num_tokens": 30649136.0, "reward": 0.6111557722091675, "reward_std": 0.19511596262454986, "rewards/ADEnReward/mean": 0.018196862610056996, "rewards/ADEnReward/std": 0.053422618098556994, "rewards/CriticEdgeReward/mean": 0.49879218339920045, "rewards/CriticEdgeReward/std": 0.176685693860054, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.30006415247917173, "sampling/importance_sampling_ratio/max": 1.9224517107009889, "sampling/importance_sampling_ratio/mean": 0.33475049436092374, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.039197158813477, "sampling/sampling_logp_difference/mean": 0.028336626663804054, "step": 275, "step_time": 17.809465072397142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 829.0, "completions/max_terminated_length": 829.0, "completions/mean_length": 734.5041870117187, "completions/mean_terminated_length": 734.5041870117187, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.6535381078720093, "epoch": 0.04200420042004201, "frac_reward_zero_std": 0.0, "grad_norm": 1.0703125, "learning_rate": 3.309352517985611e-07, "loss": -0.0472, "num_tokens": 31206505.0, "reward": 0.6943913340568543, "reward_std": 0.21788323819637298, "rewards/ADEnReward/mean": 0.04252834133803844, "rewards/ADEnReward/std": 0.09622640386223794, "rewards/CriticEdgeReward/mean": 0.5626963078975677, "rewards/CriticEdgeReward/std": 0.19119986295700073, "rewards/StrictFormatReward/mean": 0.8916666626930236, "rewards/StrictFormatReward/std": 0.431454199552536, "sampling/importance_sampling_ratio/max": 2.605127716064453, "sampling/importance_sampling_ratio/mean": 0.37264403104782107, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.030103445053101, "sampling/sampling_logp_difference/mean": 0.027391670271754265, "step": 280, "step_time": 17.91133863357827 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 925.6, "completions/max_terminated_length": 925.6, "completions/mean_length": 735.4125244140625, "completions/mean_terminated_length": 735.4125244140625, "completions/min_length": 654.6, "completions/min_terminated_length": 654.6, "entropy": 0.6702277839183808, "epoch": 0.04275427542754275, "frac_reward_zero_std": 0.0, "grad_norm": 0.8515625, "learning_rate": 3.1894484412470026e-07, "loss": -0.0602, "num_tokens": 31764124.0, "reward": 0.6529731750488281, "reward_std": 0.23486365079879762, "rewards/ADEnReward/mean": 0.0363221600651741, "rewards/ADEnReward/std": 0.07980739697813988, "rewards/CriticEdgeReward/mean": 0.5241509795188903, "rewards/CriticEdgeReward/std": 0.20597483217716217, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.37487201690673827, "sampling/importance_sampling_ratio/max": 2.7223117351531982, "sampling/importance_sampling_ratio/mean": 0.3854650676250458, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7336313247680666, "sampling/sampling_logp_difference/mean": 0.027347036823630332, "step": 285, "step_time": 18.656947684194893 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 831.6, "completions/max_terminated_length": 831.6, "completions/mean_length": 731.708349609375, "completions/mean_terminated_length": 731.708349609375, "completions/min_length": 669.4, "completions/min_terminated_length": 669.4, "entropy": 0.6643049061298371, "epoch": 0.043504350435043505, "frac_reward_zero_std": 0.0, "grad_norm": 0.72265625, "learning_rate": 3.069544364508393e-07, "loss": -0.0512, "num_tokens": 32320566.0, "reward": 0.6708286881446839, "reward_std": 0.23809046447277069, "rewards/ADEnReward/mean": 0.037316465936601165, "rewards/ADEnReward/std": 0.08676125034689904, "rewards/CriticEdgeReward/mean": 0.5360121726989746, "rewards/CriticEdgeReward/std": 0.20618329346179962, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.2079800605773925, "sampling/importance_sampling_ratio/mean": 0.3743889033794403, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5242683410644533, "sampling/sampling_logp_difference/mean": 0.026632488891482353, "step": 290, "step_time": 17.84910691343248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 799.2, "completions/max_terminated_length": 799.2, "completions/mean_length": 728.86669921875, "completions/mean_terminated_length": 728.86669921875, "completions/min_length": 677.6, "completions/min_terminated_length": 677.6, "entropy": 0.6591454565525054, "epoch": 0.04425442544254425, "frac_reward_zero_std": 0.0, "grad_norm": 0.8515625, "learning_rate": 2.949640287769784e-07, "loss": -0.0508, "num_tokens": 32876566.0, "reward": 0.6684452652931213, "reward_std": 0.23737358748912812, "rewards/ADEnReward/mean": 0.03502554502338171, "rewards/ADEnReward/std": 0.08931473270058632, "rewards/CriticEdgeReward/mean": 0.5359196782112121, "rewards/CriticEdgeReward/std": 0.20658129155635835, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.479127287864685, "sampling/importance_sampling_ratio/mean": 0.34995701909065247, "sampling/importance_sampling_ratio/min": 0.001848163455724716, "sampling/sampling_logp_difference/max": 3.5466017723083496, "sampling/sampling_logp_difference/mean": 0.02662847638130188, "step": 295, "step_time": 17.700388821028174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 942.0, "completions/max_terminated_length": 942.0, "completions/mean_length": 735.6958618164062, "completions/mean_terminated_length": 735.6958618164062, "completions/min_length": 661.6, "completions/min_terminated_length": 661.6, "entropy": 0.6566054046154022, "epoch": 0.045004500450045004, "frac_reward_zero_std": 0.0, "grad_norm": 0.310546875, "learning_rate": 2.8297362110311753e-07, "loss": -0.0027, "num_tokens": 33434125.0, "reward": 0.6631090641021729, "reward_std": 0.219744673371315, "rewards/ADEnReward/mean": 0.0353838162496686, "rewards/ADEnReward/std": 0.07947984151542187, "rewards/CriticEdgeReward/mean": 0.5352252304553986, "rewards/CriticEdgeReward/std": 0.19675078690052034, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.2790679454803467, "sampling/importance_sampling_ratio/max": 2.220416951179504, "sampling/importance_sampling_ratio/mean": 0.3625098764896393, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9535260677337645, "sampling/sampling_logp_difference/mean": 0.026643800362944604, "step": 300, "step_time": 18.494257751759143 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 962.6, "completions/max_terminated_length": 816.6, "completions/mean_length": 735.2458618164062, "completions/mean_terminated_length": 731.8783813476563, "completions/min_length": 575.2, "completions/min_terminated_length": 575.2, "entropy": 0.6507114946842194, "epoch": 0.04575457545754576, "frac_reward_zero_std": 0.0, "grad_norm": 0.44140625, "learning_rate": 2.7098321342925657e-07, "loss": -0.0384, "num_tokens": 33992072.0, "reward": 0.6791980862617493, "reward_std": 0.2274581789970398, "rewards/ADEnReward/mean": 0.01753966696560383, "rewards/ADEnReward/std": 0.0640442617237568, "rewards/CriticEdgeReward/mean": 0.5666583836078644, "rewards/CriticEdgeReward/std": 0.20808005332946777, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.22719413042068481, "sampling/importance_sampling_ratio/max": 2.364753818511963, "sampling/importance_sampling_ratio/mean": 0.3503287136554718, "sampling/importance_sampling_ratio/min": 0.0005966726690530777, "sampling/sampling_logp_difference/max": 2.855735111236572, "sampling/sampling_logp_difference/mean": 0.026694745942950248, "step": 305, "step_time": 18.11942444732413 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1029.6, "completions/max_terminated_length": 876.6, "completions/mean_length": 732.745849609375, "completions/mean_terminated_length": 729.405419921875, "completions/min_length": 548.4, "completions/min_terminated_length": 548.4, "entropy": 0.6552319288253784, "epoch": 0.0465046504650465, "frac_reward_zero_std": 0.0, "grad_norm": 0.80078125, "learning_rate": 2.5899280575539566e-07, "loss": -0.0268, "num_tokens": 34548731.0, "reward": 0.6637788057327271, "reward_std": 0.2225680559873581, "rewards/ADEnReward/mean": 0.02874856684356928, "rewards/ADEnReward/std": 0.06458092182874679, "rewards/CriticEdgeReward/mean": 0.5400302112102509, "rewards/CriticEdgeReward/std": 0.20730426907539368, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.27702280282974245, "sampling/importance_sampling_ratio/max": 2.531292176246643, "sampling/importance_sampling_ratio/mean": 0.4236529231071472, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.6013640403747558, "sampling/sampling_logp_difference/mean": 0.026283545047044755, "step": 310, "step_time": 19.1360728058964 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 847.0, "completions/max_terminated_length": 847.0, "completions/mean_length": 731.35419921875, "completions/mean_terminated_length": 731.35419921875, "completions/min_length": 665.8, "completions/min_terminated_length": 665.8, "entropy": 0.6609957098960877, "epoch": 0.047254725472547256, "frac_reward_zero_std": 0.0, "grad_norm": 0.82421875, "learning_rate": 2.4700239808153475e-07, "loss": -0.0047, "num_tokens": 35104848.0, "reward": 0.6758762955665588, "reward_std": 0.2427089899778366, "rewards/ADEnReward/mean": 0.030612951889634133, "rewards/ADEnReward/std": 0.07592607140541077, "rewards/CriticEdgeReward/mean": 0.5485966265201568, "rewards/CriticEdgeReward/std": 0.20570530295372008, "rewards/StrictFormatReward/mean": 0.9666666507720947, "rewards/StrictFormatReward/std": 0.2309401035308838, "sampling/importance_sampling_ratio/max": 2.285525894165039, "sampling/importance_sampling_ratio/mean": 0.36414429545402527, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.431345415115357, "sampling/sampling_logp_difference/mean": 0.027302127704024313, "step": 315, "step_time": 18.02225494906306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 949.8, "completions/max_terminated_length": 949.8, "completions/mean_length": 735.8625122070313, "completions/mean_terminated_length": 735.8625122070313, "completions/min_length": 658.8, "completions/min_terminated_length": 658.8, "entropy": 0.6580738365650177, "epoch": 0.048004800480048, "frac_reward_zero_std": 0.0, "grad_norm": 0.609375, "learning_rate": 2.3501199040767386e-07, "loss": -0.0202, "num_tokens": 35662607.0, "reward": 0.6721551299095154, "reward_std": 0.22274385690689086, "rewards/ADEnReward/mean": 0.024929306469857693, "rewards/ADEnReward/std": 0.048186666518449786, "rewards/CriticEdgeReward/mean": 0.5538924813270569, "rewards/CriticEdgeReward/std": 0.20744790732860566, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3111684799194336, "sampling/importance_sampling_ratio/max": 2.565552759170532, "sampling/importance_sampling_ratio/mean": 0.3123017966747284, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.9164583683013916, "sampling/sampling_logp_difference/mean": 0.027466055005788803, "step": 320, "step_time": 18.586662534438073 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 977.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 735.3041748046875, "completions/mean_terminated_length": 735.3041748046875, "completions/min_length": 663.0, "completions/min_terminated_length": 663.0, "entropy": 0.6779762625694274, "epoch": 0.048754875487548754, "frac_reward_zero_std": 0.0, "grad_norm": 0.7265625, "learning_rate": 2.2302158273381293e-07, "loss": -0.0384, "num_tokens": 36219832.0, "reward": 0.6317761659622192, "reward_std": 0.22682821452617646, "rewards/ADEnReward/mean": 0.03518758714199066, "rewards/ADEnReward/std": 0.08642232343554497, "rewards/CriticEdgeReward/mean": 0.5007552146911621, "rewards/CriticEdgeReward/std": 0.19146932959556578, "rewards/StrictFormatReward/mean": 0.9583333373069763, "rewards/StrictFormatReward/std": 0.21928777694702148, "sampling/importance_sampling_ratio/max": 2.232274961471558, "sampling/importance_sampling_ratio/mean": 0.35539286136627196, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.039138269424439, "sampling/sampling_logp_difference/mean": 0.02667497843503952, "step": 325, "step_time": 18.764488724060357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 833.6, "completions/max_terminated_length": 833.6, "completions/mean_length": 733.8041870117188, "completions/mean_terminated_length": 733.8041870117188, "completions/min_length": 664.4, "completions/min_terminated_length": 664.4, "entropy": 0.6687864780426025, "epoch": 0.04950495049504951, "frac_reward_zero_std": 0.0, "grad_norm": 0.39453125, "learning_rate": 2.1103117505995202e-07, "loss": 0.0118, "num_tokens": 36777273.0, "reward": 0.6627076983451843, "reward_std": 0.22386887073516845, "rewards/ADEnReward/mean": 0.02126012146472931, "rewards/ADEnReward/std": 0.05546980947256088, "rewards/CriticEdgeReward/mean": 0.5506142199039459, "rewards/CriticEdgeReward/std": 0.21675859093666078, "rewards/StrictFormatReward/mean": 0.9083333253860474, "rewards/StrictFormatReward/std": 0.40901769399642945, "sampling/importance_sampling_ratio/max": 2.256238555908203, "sampling/importance_sampling_ratio/mean": 0.36524984836578367, "sampling/importance_sampling_ratio/min": 0.0008549375459551811, "sampling/sampling_logp_difference/max": 4.8658998012542725, "sampling/sampling_logp_difference/mean": 0.027810844406485558, "step": 330, "step_time": 17.94678303701803 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 973.6, "completions/max_terminated_length": 833.2, "completions/mean_length": 735.0625122070312, "completions/mean_terminated_length": 731.6976196289063, "completions/min_length": 545.6, "completions/min_terminated_length": 545.6, "entropy": 0.6478756308555603, "epoch": 0.05025502550255025, "frac_reward_zero_std": 0.0, "grad_norm": 0.49609375, "learning_rate": 1.9904076738609113e-07, "loss": 0.0046, "num_tokens": 37334760.0, "reward": 0.6804102182388305, "reward_std": 0.22279894351959229, "rewards/ADEnReward/mean": 0.02531726723536849, "rewards/ADEnReward/std": 0.06201638840138912, "rewards/CriticEdgeReward/mean": 0.5617595791816712, "rewards/CriticEdgeReward/std": 0.20368576049804688, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3079705059528351, "sampling/importance_sampling_ratio/max": 1.990720510482788, "sampling/importance_sampling_ratio/mean": 0.32842206954956055, "sampling/importance_sampling_ratio/min": 0.0019439553841948509, "sampling/sampling_logp_difference/max": 3.133005714416504, "sampling/sampling_logp_difference/mean": 0.027392688021063805, "step": 335, "step_time": 18.717536420933904 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 801.8, "completions/max_terminated_length": 801.8, "completions/mean_length": 733.0833618164063, "completions/mean_terminated_length": 733.0833618164063, "completions/min_length": 683.0, "completions/min_terminated_length": 683.0, "entropy": 0.6708641886711121, "epoch": 0.051005100510051006, "frac_reward_zero_std": 0.0, "grad_norm": 0.8046875, "learning_rate": 1.870503597122302e-07, "loss": 0.0034, "num_tokens": 37891852.0, "reward": 0.6578708171844483, "reward_std": 0.18905953764915467, "rewards/ADEnReward/mean": 0.018444513343274592, "rewards/ADEnReward/std": 0.039897975884377955, "rewards/CriticEdgeReward/mean": 0.5427596151828766, "rewards/CriticEdgeReward/std": 0.18363151252269744, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.352165174484253, "sampling/importance_sampling_ratio/mean": 0.36960681080818175, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.422780179977417, "sampling/sampling_logp_difference/mean": 0.027085694670677184, "step": 340, "step_time": 17.873688333295284 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.0, "completions/max_terminated_length": 802.0, "completions/mean_length": 732.4250244140625, "completions/mean_terminated_length": 732.4250244140625, "completions/min_length": 672.6, "completions/min_terminated_length": 672.6, "entropy": 0.6689311444759369, "epoch": 0.05175517551755175, "frac_reward_zero_std": 0.0, "grad_norm": 1.0625, "learning_rate": 1.750599520383693e-07, "loss": -0.0509, "num_tokens": 38448706.0, "reward": 0.6395164728164673, "reward_std": 0.21987482607364656, "rewards/ADEnReward/mean": 0.027375983633100988, "rewards/ADEnReward/std": 0.07665511220693588, "rewards/CriticEdgeReward/mean": 0.516307145357132, "rewards/CriticEdgeReward/std": 0.20652512311935425, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.21331926584243774, "sampling/importance_sampling_ratio/max": 2.33897225856781, "sampling/importance_sampling_ratio/mean": 0.34371902942657473, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.346931266784668, "sampling/sampling_logp_difference/mean": 0.02687378004193306, "step": 345, "step_time": 17.670364321395756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.01250000037252903, "completions/max_length": 1118.2, "completions/max_terminated_length": 823.4, "completions/mean_length": 740.270849609375, "completions/mean_terminated_length": 730.1566040039063, "completions/min_length": 633.6, "completions/min_terminated_length": 633.6, "entropy": 0.6430355548858643, "epoch": 0.052505250525052505, "frac_reward_zero_std": 0.0, "grad_norm": 0.77734375, "learning_rate": 1.630695443645084e-07, "loss": -0.0426, "num_tokens": 39007571.0, "reward": 0.6766586542129517, "reward_std": 0.24121854305267335, "rewards/ADEnReward/mean": 0.04179625120013952, "rewards/ADEnReward/std": 0.08900657370686531, "rewards/CriticEdgeReward/mean": 0.5456956803798676, "rewards/CriticEdgeReward/std": 0.20641822814941407, "rewards/StrictFormatReward/mean": 0.8916666746139527, "rewards/StrictFormatReward/std": 0.4519024074077606, "sampling/importance_sampling_ratio/max": 2.6239418506622316, "sampling/importance_sampling_ratio/mean": 0.39805959463119506, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.128032374382019, "sampling/sampling_logp_difference/mean": 0.026333746314048768, "step": 350, "step_time": 19.792572408355774 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 968.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 732.558349609375, "completions/mean_terminated_length": 732.558349609375, "completions/min_length": 658.2, "completions/min_terminated_length": 658.2, "entropy": 0.6749980449676514, "epoch": 0.05325532553255326, "frac_reward_zero_std": 0.0, "grad_norm": 0.326171875, "learning_rate": 1.5107913669064747e-07, "loss": -0.0269, "num_tokens": 39563913.0, "reward": 0.6482107043266296, "reward_std": 0.1990095168352127, "rewards/ADEnReward/mean": 0.01931877192109823, "rewards/ADEnReward/std": 0.04623195445165038, "rewards/CriticEdgeReward/mean": 0.5355585813522339, "rewards/CriticEdgeReward/std": 0.18209150433540344, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.2732768297195435, "sampling/importance_sampling_ratio/max": 1.9461352109909058, "sampling/importance_sampling_ratio/mean": 0.3097763121128082, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0171242713928224, "sampling/sampling_logp_difference/mean": 0.026656011864542962, "step": 355, "step_time": 18.716575593315063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 812.0, "completions/max_terminated_length": 812.0, "completions/mean_length": 733.433349609375, "completions/mean_terminated_length": 733.433349609375, "completions/min_length": 647.4, "completions/min_terminated_length": 647.4, "entropy": 0.6696459889411926, "epoch": 0.054005400540054004, "frac_reward_zero_std": 0.0, "grad_norm": 0.703125, "learning_rate": 1.3908872901678656e-07, "loss": -0.0327, "num_tokens": 40121073.0, "reward": 0.6496767163276672, "reward_std": 0.2308586984872818, "rewards/ADEnReward/mean": 0.03774411976337433, "rewards/ADEnReward/std": 0.08999090753495693, "rewards/CriticEdgeReward/mean": 0.5169325530529022, "rewards/CriticEdgeReward/std": 0.19961123764514924, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.31171647906303407, "sampling/importance_sampling_ratio/max": 2.5782790184020996, "sampling/importance_sampling_ratio/mean": 0.357247918844223, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.792746257781983, "sampling/sampling_logp_difference/mean": 0.026708434894680978, "step": 360, "step_time": 17.821264788974077 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 931.4, "completions/max_terminated_length": 931.4, "completions/mean_length": 733.3458740234375, "completions/mean_terminated_length": 733.3458740234375, "completions/min_length": 669.4, "completions/min_terminated_length": 669.4, "entropy": 0.6525244832038879, "epoch": 0.054755475547554756, "frac_reward_zero_std": 0.0, "grad_norm": 0.41796875, "learning_rate": 1.2709832134292565e-07, "loss": -0.0169, "num_tokens": 40678132.0, "reward": 0.6920304536819458, "reward_std": 0.20770387947559357, "rewards/ADEnReward/mean": 0.02554720304906368, "rewards/ADEnReward/std": 0.058688711002469064, "rewards/CriticEdgeReward/mean": 0.5689831852912903, "rewards/CriticEdgeReward/std": 0.19076889455318452, "rewards/StrictFormatReward/mean": 0.975, "rewards/StrictFormatReward/std": 0.13851140141487123, "sampling/importance_sampling_ratio/max": 2.144974970817566, "sampling/importance_sampling_ratio/mean": 0.3225821018218994, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.571043276786804, "sampling/sampling_logp_difference/mean": 0.026927465572953224, "step": 365, "step_time": 18.449670459423213 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 958.2, "completions/max_terminated_length": 820.0, "completions/mean_length": 734.9208618164063, "completions/mean_terminated_length": 731.5512573242188, "completions/min_length": 574.4, "completions/min_terminated_length": 574.4, "entropy": 0.6623149275779724, "epoch": 0.0555055505550555, "frac_reward_zero_std": 0.0, "grad_norm": 0.69140625, "learning_rate": 1.1510791366906475e-07, "loss": -0.0306, "num_tokens": 41235681.0, "reward": 0.6599913954734802, "reward_std": 0.24132562279701233, "rewards/ADEnReward/mean": 0.0372123297303915, "rewards/ADEnReward/std": 0.09461619928479195, "rewards/CriticEdgeReward/mean": 0.5294457077980042, "rewards/CriticEdgeReward/std": 0.19925108551979065, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3111684799194336, "sampling/importance_sampling_ratio/max": 2.391965961456299, "sampling/importance_sampling_ratio/mean": 0.4664926826953888, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.167402553558349, "sampling/sampling_logp_difference/mean": 0.026942481473088264, "step": 370, "step_time": 18.869643361121415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 944.2, "completions/max_terminated_length": 944.2, "completions/mean_length": 731.5250244140625, "completions/mean_terminated_length": 731.5250244140625, "completions/min_length": 555.6, "completions/min_terminated_length": 555.6, "entropy": 0.6653449892997741, "epoch": 0.056255625562556255, "frac_reward_zero_std": 0.0, "grad_norm": 0.42578125, "learning_rate": 1.0311750599520383e-07, "loss": -0.0691, "num_tokens": 41792367.0, "reward": 0.6625326037406921, "reward_std": 0.23414765000343324, "rewards/ADEnReward/mean": 0.025838583149015902, "rewards/ADEnReward/std": 0.05779963061213493, "rewards/CriticEdgeReward/mean": 0.5416939973831176, "rewards/CriticEdgeReward/std": 0.21254521906375884, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.22719413042068481, "sampling/importance_sampling_ratio/max": 2.27764630317688, "sampling/importance_sampling_ratio/mean": 0.3599185526371002, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.140010356903076, "sampling/sampling_logp_difference/mean": 0.027480918169021606, "step": 375, "step_time": 18.708393464796245 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 820.4, "completions/max_terminated_length": 820.4, "completions/mean_length": 727.9541870117188, "completions/mean_terminated_length": 727.9541870117188, "completions/min_length": 652.2, "completions/min_terminated_length": 652.2, "entropy": 0.6534910440444947, "epoch": 0.05700570057005701, "frac_reward_zero_std": 0.0, "grad_norm": 0.68359375, "learning_rate": 9.112709832134293e-08, "loss": 0.0054, "num_tokens": 42348004.0, "reward": 0.6869569540023803, "reward_std": 0.24174749255180358, "rewards/ADEnReward/mean": 0.03954169936478138, "rewards/ADEnReward/std": 0.08550861254334449, "rewards/CriticEdgeReward/mean": 0.5549151957035064, "rewards/CriticEdgeReward/std": 0.20269282758235932, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.3342098295688629, "sampling/importance_sampling_ratio/max": 2.566698694229126, "sampling/importance_sampling_ratio/mean": 0.4260771095752716, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.149584627151489, "sampling/sampling_logp_difference/mean": 0.026491811498999596, "step": 380, "step_time": 17.704850973002614 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1077.8, "completions/max_terminated_length": 968.0, "completions/mean_length": 741.6583618164062, "completions/mean_terminated_length": 738.3051635742188, "completions/min_length": 673.2, "completions/min_terminated_length": 673.2, "entropy": 0.6719724714756012, "epoch": 0.057755775577557754, "frac_reward_zero_std": 0.0, "grad_norm": 0.67578125, "learning_rate": 7.913669064748201e-08, "loss": -0.0432, "num_tokens": 42907042.0, "reward": 0.6660396218299866, "reward_std": 0.22775462567806243, "rewards/ADEnReward/mean": 0.0374774681404233, "rewards/ADEnReward/std": 0.07977748811244964, "rewards/CriticEdgeReward/mean": 0.5343954741954804, "rewards/CriticEdgeReward/std": 0.1990694969892502, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.30006415247917173, "sampling/importance_sampling_ratio/max": 2.322414183616638, "sampling/importance_sampling_ratio/mean": 0.4185632526874542, "sampling/importance_sampling_ratio/min": 0.0005454799626022578, "sampling/sampling_logp_difference/max": 3.8949396133422853, "sampling/sampling_logp_difference/mean": 0.027754966542124747, "step": 385, "step_time": 19.44117357665673 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 948.6, "completions/max_terminated_length": 808.0, "completions/mean_length": 733.7916870117188, "completions/mean_terminated_length": 730.452587890625, "completions/min_length": 666.0, "completions/min_terminated_length": 666.0, "entropy": 0.6774083852767945, "epoch": 0.05850585058505851, "frac_reward_zero_std": 0.0, "grad_norm": 0.56640625, "learning_rate": 6.71462829736211e-08, "loss": -0.0345, "num_tokens": 43464368.0, "reward": 0.6508362531661988, "reward_std": 0.20807888507843017, "rewards/ADEnReward/mean": 0.028987116180360317, "rewards/ADEnReward/std": 0.07207153961062432, "rewards/CriticEdgeReward/mean": 0.5243491053581237, "rewards/CriticEdgeReward/std": 0.19485931992530822, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.213830828666687, "sampling/importance_sampling_ratio/mean": 0.374602335691452, "sampling/importance_sampling_ratio/min": 0.001983413042034954, "sampling/sampling_logp_difference/max": 3.6328057765960695, "sampling/sampling_logp_difference/mean": 0.026926656439900398, "step": 390, "step_time": 18.684992133639753 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 935.4, "completions/max_terminated_length": 935.4, "completions/mean_length": 735.5708618164062, "completions/mean_terminated_length": 735.5708618164062, "completions/min_length": 661.0, "completions/min_terminated_length": 661.0, "entropy": 0.667855304479599, "epoch": 0.05925592559255925, "frac_reward_zero_std": 0.0, "grad_norm": 0.97265625, "learning_rate": 5.5155875299760186e-08, "loss": -0.0054, "num_tokens": 44021993.0, "reward": 0.6574560403823853, "reward_std": 0.2127603441476822, "rewards/ADEnReward/mean": 0.02033945359289646, "rewards/ADEnReward/std": 0.04808502271771431, "rewards/CriticEdgeReward/mean": 0.5429499030113221, "rewards/CriticEdgeReward/std": 0.1970933496952057, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.6854350090026857, "sampling/importance_sampling_ratio/mean": 0.363179612159729, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.153783226013184, "sampling/sampling_logp_difference/mean": 0.02747395932674408, "step": 395, "step_time": 18.734802085906267 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 814.6, "completions/max_terminated_length": 814.6, "completions/mean_length": 728.6625122070312, "completions/mean_terminated_length": 728.6625122070312, "completions/min_length": 559.0, "completions/min_terminated_length": 559.0, "entropy": 0.6656871378421784, "epoch": 0.060006000600060005, "frac_reward_zero_std": 0.0, "grad_norm": 0.84765625, "learning_rate": 4.316546762589928e-08, "loss": -0.0489, "num_tokens": 44578088.0, "reward": 0.7010002613067627, "reward_std": 0.2456672489643097, "rewards/ADEnReward/mean": 0.04901094548404217, "rewards/ADEnReward/std": 0.09055124968290329, "rewards/CriticEdgeReward/mean": 0.5586559414863587, "rewards/CriticEdgeReward/std": 0.21144209206104278, "rewards/StrictFormatReward/mean": 0.9333333134651184, "rewards/StrictFormatReward/std": 0.3426641821861267, "sampling/importance_sampling_ratio/max": 2.1861767053604124, "sampling/importance_sampling_ratio/mean": 0.32495183646678927, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.1166597366333, "sampling/sampling_logp_difference/mean": 0.02740965746343136, "step": 400, "step_time": 17.950684045162053 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 948.6, "completions/max_terminated_length": 800.0, "completions/mean_length": 735.0500244140625, "completions/mean_terminated_length": 731.7193359375, "completions/min_length": 672.8, "completions/min_terminated_length": 672.8, "entropy": 0.6734409689903259, "epoch": 0.06075607560756076, "frac_reward_zero_std": 0.0, "grad_norm": 0.98828125, "learning_rate": 3.1175059952038366e-08, "loss": -0.0654, "num_tokens": 45135364.0, "reward": 0.6133486986160278, "reward_std": 0.22886175513267518, "rewards/ADEnReward/mean": 0.025396238639950754, "rewards/ADEnReward/std": 0.06664117574691772, "rewards/CriticEdgeReward/mean": 0.49128576517105105, "rewards/CriticEdgeReward/std": 0.20492425858974456, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.3328298330307007, "sampling/importance_sampling_ratio/mean": 0.3836908459663391, "sampling/importance_sampling_ratio/min": 0.00026381765492260455, "sampling/sampling_logp_difference/max": 2.4349581241607665, "sampling/sampling_logp_difference/mean": 0.027349038049578667, "step": 405, "step_time": 18.79958414826542 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 928.4, "completions/max_terminated_length": 928.4, "completions/mean_length": 734.6875366210937, "completions/mean_terminated_length": 734.6875366210937, "completions/min_length": 668.4, "completions/min_terminated_length": 668.4, "entropy": 0.6596668481826782, "epoch": 0.061506150615061504, "frac_reward_zero_std": 0.0, "grad_norm": 0.73046875, "learning_rate": 1.9184652278177456e-08, "loss": -0.0135, "num_tokens": 45692697.0, "reward": 0.643831217288971, "reward_std": 0.2153584510087967, "rewards/ADEnReward/mean": 0.019771945616230368, "rewards/ADEnReward/std": 0.060220534168183806, "rewards/CriticEdgeReward/mean": 0.529059213399887, "rewards/CriticEdgeReward/std": 0.20209163427352905, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.22719413042068481, "sampling/importance_sampling_ratio/max": 2.006116271018982, "sampling/importance_sampling_ratio/mean": 0.3227749466896057, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.683455276489258, "sampling/sampling_logp_difference/mean": 0.027268433570861818, "step": 410, "step_time": 18.420727408491075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 822.8, "completions/max_terminated_length": 822.8, "completions/mean_length": 733.1375244140625, "completions/mean_terminated_length": 733.1375244140625, "completions/min_length": 680.8, "completions/min_terminated_length": 680.8, "entropy": 0.6543232202529907, "epoch": 0.06225622562256226, "frac_reward_zero_std": 0.0, "grad_norm": 0.69921875, "learning_rate": 7.194244604316547e-09, "loss": 0.0231, "num_tokens": 46249834.0, "reward": 0.6894648790359497, "reward_std": 0.2503668010234833, "rewards/ADEnReward/mean": 0.04139488632790744, "rewards/ADEnReward/std": 0.09687295109033585, "rewards/CriticEdgeReward/mean": 0.5497366309165954, "rewards/CriticEdgeReward/std": 0.2105077773332596, "rewards/StrictFormatReward/mean": 0.9833333253860473, "rewards/StrictFormatReward/std": 0.1154700517654419, "sampling/importance_sampling_ratio/max": 2.1611329317092896, "sampling/importance_sampling_ratio/mean": 0.3071496218442917, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1998504638671874, "sampling/sampling_logp_difference/mean": 0.027152119576931, "step": 415, "step_time": 17.82053266847506 } ], "logging_steps": 5, "max_steps": 417, "num_input_tokens_seen": 46473126, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 6, "trial_name": null, "trial_params": null }