{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.06255625562556255, "eval_steps": 100, "global_step": 417, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 813.4, "completions/max_terminated_length": 813.4, "completions/mean_length": 732.3125244140625, "completions/mean_terminated_length": 732.3125244140625, "completions/min_length": 547.6, "completions/min_terminated_length": 547.6, "entropy": 0.6945622801780701, "epoch": 0.00075007500750075, "frac_reward_zero_std": 0.0, "grad_norm": 0.71484375, "learning_rate": 9.904076738609112e-07, "loss": -0.0662, "num_tokens": 556763.0, "reward": 0.25436362624168396, "reward_std": 0.22843295931816102, "rewards/ADEnReward/mean": 0.03650980964303017, "rewards/ADEnReward/std": 0.0942004919052124, "rewards/CriticV2Reward/mean": 0.12452046498656273, "rewards/CriticV2Reward/std": 0.2022954449057579, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.736471652984619, "sampling/importance_sampling_ratio/mean": 0.42077494263648985, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.499092960357666, "sampling/sampling_logp_difference/mean": 0.027753319963812827, "step": 5, "step_time": 16.534613192547113 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 803.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 733.6416870117188, "completions/mean_terminated_length": 733.6416870117188, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "entropy": 0.6582455694675445, "epoch": 0.0015001500150015, "frac_reward_zero_std": 0.0, "grad_norm": 1.3203125, "learning_rate": 9.784172661870503e-07, "loss": -0.0192, "num_tokens": 1114181.0, "reward": 0.2501808494329453, "reward_std": 0.22661161720752715, "rewards/ADEnReward/mean": 0.01843893507029861, "rewards/ADEnReward/std": 0.04499729741364718, "rewards/CriticV2Reward/mean": 0.13840857446193694, "rewards/CriticV2Reward/std": 0.21599454879760743, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.801686000823975, "sampling/importance_sampling_ratio/mean": 0.4209885835647583, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.701645183563232, "sampling/sampling_logp_difference/mean": 0.027708011493086816, "step": 10, "step_time": 13.271006786078214 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 963.6, "completions/max_terminated_length": 963.6, "completions/mean_length": 738.40419921875, "completions/mean_terminated_length": 738.40419921875, "completions/min_length": 645.2, "completions/min_terminated_length": 645.2, "entropy": 0.6592447817325592, "epoch": 0.0022502250225022503, "frac_reward_zero_std": 0.0, "grad_norm": 0.8359375, "learning_rate": 9.664268585131893e-07, "loss": 0.0102, "num_tokens": 1672550.0, "reward": 0.3058495342731476, "reward_std": 0.2918362975120544, "rewards/ADEnReward/mean": 0.04017725773155689, "rewards/ADEnReward/std": 0.09973704516887665, "rewards/CriticV2Reward/mean": 0.1756722643971443, "rewards/CriticV2Reward/std": 0.2586840629577637, "rewards/StrictFormatReward/mean": 0.9000000119209289, "rewards/StrictFormatReward/std": 0.434829568862915, "sampling/importance_sampling_ratio/max": 2.4545116901397703, "sampling/importance_sampling_ratio/mean": 0.37839311361312866, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.780246067047119, "sampling/sampling_logp_difference/mean": 0.027404751628637314, "step": 15, "step_time": 14.447486751060932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 826.6, "completions/max_terminated_length": 826.6, "completions/mean_length": 734.4125244140625, "completions/mean_terminated_length": 734.4125244140625, "completions/min_length": 583.4, "completions/min_terminated_length": 583.4, "entropy": 0.6629917621612549, "epoch": 0.003000300030003, "frac_reward_zero_std": 0.0, "grad_norm": 0.3359375, "learning_rate": 9.544364508393285e-07, "loss": -0.0869, "num_tokens": 2230217.0, "reward": 0.29596259295940397, "reward_std": 0.2525408238172531, "rewards/ADEnReward/mean": 0.03129123505204916, "rewards/ADEnReward/std": 0.07233726158738137, "rewards/CriticV2Reward/mean": 0.16800468116998674, "rewards/CriticV2Reward/std": 0.23792918026447296, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.1899476528167723, "sampling/importance_sampling_ratio/mean": 0.3364464044570923, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7939449310302735, "sampling/sampling_logp_difference/mean": 0.0276542279869318, "step": 20, "step_time": 13.247872665338218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 927.4, "completions/max_terminated_length": 927.4, "completions/mean_length": 733.54169921875, "completions/mean_terminated_length": 733.54169921875, "completions/min_length": 668.4, "completions/min_terminated_length": 668.4, "entropy": 0.6701843202114105, "epoch": 0.0037503750375037503, "frac_reward_zero_std": 0.0, "grad_norm": 0.65625, "learning_rate": 9.424460431654676e-07, "loss": -0.0358, "num_tokens": 2787211.0, "reward": 0.33442996740341185, "reward_std": 0.30991793870925904, "rewards/ADEnReward/mean": 0.055364159494638444, "rewards/ADEnReward/std": 0.11989772319793701, "rewards/CriticV2Reward/mean": 0.1823991283774376, "rewards/CriticV2Reward/std": 0.26879840195178983, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.1555842399597168, "sampling/importance_sampling_ratio/max": 2.724516677856445, "sampling/importance_sampling_ratio/mean": 0.4627986252307892, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.788603591918945, "sampling/sampling_logp_difference/mean": 0.026526946946978568, "step": 25, "step_time": 13.950103689264505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 855.2, "completions/max_terminated_length": 855.2, "completions/mean_length": 736.0041870117187, "completions/mean_terminated_length": 736.0041870117187, "completions/min_length": 653.4, "completions/min_terminated_length": 653.4, "entropy": 0.6765582263469696, "epoch": 0.004500450045004501, "frac_reward_zero_std": 0.0, "grad_norm": 1.53125, "learning_rate": 9.304556354916066e-07, "loss": -0.0296, "num_tokens": 3344860.0, "reward": 0.3146974563598633, "reward_std": 0.2686339944601059, "rewards/ADEnReward/mean": 0.02965494096279144, "rewards/ADEnReward/std": 0.07100159302353859, "rewards/CriticV2Reward/mean": 0.19254249930381775, "rewards/CriticV2Reward/std": 0.2526032477617264, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.3250433325767517, "sampling/importance_sampling_ratio/max": 2.5650532245635986, "sampling/importance_sampling_ratio/mean": 0.410943877696991, "sampling/importance_sampling_ratio/min": 0.00010549233993515373, "sampling/sampling_logp_difference/max": 4.272568225860596, "sampling/sampling_logp_difference/mean": 0.027386844530701636, "step": 30, "step_time": 13.507405268307775 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 962.8, "completions/max_terminated_length": 815.8, "completions/mean_length": 740.1208618164062, "completions/mean_terminated_length": 736.7608276367188, "completions/min_length": 669.6, "completions/min_terminated_length": 669.6, "entropy": 0.6634912610054016, "epoch": 0.005250525052505251, "frac_reward_zero_std": 0.0, "grad_norm": 0.447265625, "learning_rate": 9.184652278177458e-07, "loss": 0.0304, "num_tokens": 3903929.0, "reward": 0.284199857711792, "reward_std": 0.2560150265693665, "rewards/ADEnReward/mean": 0.02072476325556636, "rewards/ADEnReward/std": 0.053264892846345904, "rewards/CriticV2Reward/mean": 0.1701417475938797, "rewards/CriticV2Reward/std": 0.24270081222057344, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.2670124769210815, "sampling/importance_sampling_ratio/mean": 0.3484788566827774, "sampling/importance_sampling_ratio/min": 4.5519565901486205e-06, "sampling/sampling_logp_difference/max": 4.598708057403565, "sampling/sampling_logp_difference/mean": 0.02718786969780922, "step": 35, "step_time": 14.319740631990134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 933.0, "completions/max_terminated_length": 933.0, "completions/mean_length": 741.645849609375, "completions/mean_terminated_length": 741.645849609375, "completions/min_length": 667.4, "completions/min_terminated_length": 667.4, "entropy": 0.6647457540035248, "epoch": 0.006000600060006, "frac_reward_zero_std": 0.0, "grad_norm": 0.84765625, "learning_rate": 9.064748201438849e-07, "loss": -0.0366, "num_tokens": 4463124.0, "reward": 0.29417389035224917, "reward_std": 0.26714652180671694, "rewards/ADEnReward/mean": 0.03201424721628428, "rewards/ADEnReward/std": 0.08157214596867561, "rewards/CriticV2Reward/mean": 0.16799295991659163, "rewards/CriticV2Reward/std": 0.246389901638031, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.2125140190124513, "sampling/importance_sampling_ratio/mean": 0.35879886746406553, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.105627155303955, "sampling/sampling_logp_difference/mean": 0.027599729597568512, "step": 40, "step_time": 14.012726863101125 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1093.6, "completions/max_terminated_length": 939.8, "completions/mean_length": 735.833349609375, "completions/mean_terminated_length": 732.4203979492188, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "entropy": 0.6987628817558289, "epoch": 0.0067506750675067504, "frac_reward_zero_std": 0.0, "grad_norm": 1.09375, "learning_rate": 8.944844124700239e-07, "loss": -0.0326, "num_tokens": 5020732.0, "reward": 0.2718124330043793, "reward_std": 0.27098544836044314, "rewards/ADEnReward/mean": 0.030245957523584367, "rewards/ADEnReward/std": 0.07657596468925476, "rewards/CriticV2Reward/mean": 0.15239980369806289, "rewards/CriticV2Reward/std": 0.23993462324142456, "rewards/StrictFormatReward/mean": 0.8916666626930236, "rewards/StrictFormatReward/std": 0.44593390822410583, "sampling/importance_sampling_ratio/max": 2.7716786861419678, "sampling/importance_sampling_ratio/mean": 0.37447538375854494, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.8250603675842285, "sampling/sampling_logp_difference/mean": 0.028124269843101502, "step": 45, "step_time": 14.862587895337493 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 824.8, "completions/max_terminated_length": 824.8, "completions/mean_length": 733.1625122070312, "completions/mean_terminated_length": 733.1625122070312, "completions/min_length": 667.4, "completions/min_terminated_length": 667.4, "entropy": 0.6778076231479645, "epoch": 0.007500750075007501, "frac_reward_zero_std": 0.0, "grad_norm": 0.54296875, "learning_rate": 8.82494004796163e-07, "loss": -0.0308, "num_tokens": 5577763.0, "reward": 0.29854954183101656, "reward_std": 0.256597700715065, "rewards/ADEnReward/mean": 0.03557878658175469, "rewards/ADEnReward/std": 0.0968101128935814, "rewards/CriticV2Reward/mean": 0.16547072678804398, "rewards/CriticV2Reward/std": 0.23895596265792846, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.2660447359085083, "sampling/importance_sampling_ratio/mean": 0.37205352783203127, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8447646379470823, "sampling/sampling_logp_difference/mean": 0.027005545049905776, "step": 50, "step_time": 13.36007147487253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 831.8, "completions/max_terminated_length": 831.8, "completions/mean_length": 730.0625244140625, "completions/mean_terminated_length": 730.0625244140625, "completions/min_length": 428.4, "completions/min_terminated_length": 428.4, "entropy": 0.6709417462348938, "epoch": 0.00825082508250825, "frac_reward_zero_std": 0.0, "grad_norm": 0.734375, "learning_rate": 8.705035971223021e-07, "loss": -0.0405, "num_tokens": 6134242.0, "reward": 0.3022352814674377, "reward_std": 0.28455966114997866, "rewards/ADEnReward/mean": 0.02506868541240692, "rewards/ADEnReward/std": 0.0622736057266593, "rewards/CriticV2Reward/mean": 0.18383326530456542, "rewards/CriticV2Reward/std": 0.2717557460069656, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3111684799194336, "sampling/importance_sampling_ratio/max": 2.504676914215088, "sampling/importance_sampling_ratio/mean": 0.35438522696495056, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3409300088882445, "sampling/sampling_logp_difference/mean": 0.028006478771567346, "step": 55, "step_time": 13.314154567103833 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1091.6, "completions/max_terminated_length": 1091.6, "completions/mean_length": 740.2875122070312, "completions/mean_terminated_length": 740.2875122070312, "completions/min_length": 645.2, "completions/min_terminated_length": 645.2, "entropy": 0.6656760215759278, "epoch": 0.009000900090009001, "frac_reward_zero_std": 0.0, "grad_norm": 0.43359375, "learning_rate": 8.585131894484412e-07, "loss": 0.0216, "num_tokens": 6692823.0, "reward": 0.2957646012306213, "reward_std": 0.2834887892007828, "rewards/ADEnReward/mean": 0.023848682641983032, "rewards/ADEnReward/std": 0.06435841396450996, "rewards/CriticV2Reward/mean": 0.1802492320537567, "rewards/CriticV2Reward/std": 0.26042196750640867, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.39194485545158386, "sampling/importance_sampling_ratio/max": 1.8143912553787231, "sampling/importance_sampling_ratio/mean": 0.30097441375255585, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.154077529907227, "sampling/sampling_logp_difference/mean": 0.027227402478456498, "step": 60, "step_time": 15.13567282371223 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 876.8, "completions/max_terminated_length": 876.8, "completions/mean_length": 734.7583618164062, "completions/mean_terminated_length": 734.7583618164062, "completions/min_length": 662.8, "completions/min_terminated_length": 662.8, "entropy": 0.6781624317169189, "epoch": 0.00975097509750975, "frac_reward_zero_std": 0.0, "grad_norm": 0.271484375, "learning_rate": 8.465227817745802e-07, "loss": -0.0528, "num_tokens": 7250205.0, "reward": 0.2756860703229904, "reward_std": 0.23353091776371002, "rewards/ADEnReward/mean": 0.03827684186398983, "rewards/ADEnReward/std": 0.07406068071722985, "rewards/CriticV2Reward/mean": 0.1440758764743805, "rewards/CriticV2Reward/std": 0.21903067827224731, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.2966887950897217, "sampling/importance_sampling_ratio/max": 2.4248700618743895, "sampling/importance_sampling_ratio/mean": 0.34985809326171874, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2400750637054445, "sampling/sampling_logp_difference/mean": 0.027471666783094408, "step": 65, "step_time": 13.655441191699357 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1133.4, "completions/max_terminated_length": 985.8, "completions/mean_length": 740.595849609375, "completions/mean_terminated_length": 737.2568237304688, "completions/min_length": 664.0, "completions/min_terminated_length": 664.0, "entropy": 0.6988117396831512, "epoch": 0.010501050105010502, "frac_reward_zero_std": 0.0, "grad_norm": 0.6484375, "learning_rate": 8.345323741007194e-07, "loss": -0.059, "num_tokens": 7808828.0, "reward": 0.22612838745117186, "reward_std": 0.19224123358726503, "rewards/ADEnReward/mean": 0.020160195766948164, "rewards/ADEnReward/std": 0.04457876272499561, "rewards/CriticV2Reward/mean": 0.11180151477456093, "rewards/CriticV2Reward/std": 0.18313901424407958, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.2685441970825195, "sampling/importance_sampling_ratio/mean": 0.3226285606622696, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.104148912429809, "sampling/sampling_logp_difference/mean": 0.027278991416096687, "step": 70, "step_time": 15.464104375429452 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 937.2, "completions/max_terminated_length": 937.2, "completions/mean_length": 735.0416870117188, "completions/mean_terminated_length": 735.0416870117188, "completions/min_length": 660.0, "completions/min_terminated_length": 660.0, "entropy": 0.6734846651554107, "epoch": 0.011251125112511251, "frac_reward_zero_std": 0.0, "grad_norm": 0.8671875, "learning_rate": 8.225419664268585e-07, "loss": 0.0331, "num_tokens": 8366182.0, "reward": 0.2327167123556137, "reward_std": 0.20908381342887877, "rewards/ADEnReward/mean": 0.02178483735769987, "rewards/ADEnReward/std": 0.0545375794172287, "rewards/CriticV2Reward/mean": 0.11926519349217415, "rewards/CriticV2Reward/std": 0.18717683404684066, "rewards/StrictFormatReward/mean": 0.9166666746139527, "rewards/StrictFormatReward/std": 0.34277149438858034, "sampling/importance_sampling_ratio/max": 2.570010709762573, "sampling/importance_sampling_ratio/mean": 0.41035584211349485, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5141350746154787, "sampling/sampling_logp_difference/mean": 0.027138520777225495, "step": 75, "step_time": 14.170246110949666 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 825.2, "completions/max_terminated_length": 825.2, "completions/mean_length": 731.9916870117188, "completions/mean_terminated_length": 731.9916870117188, "completions/min_length": 659.0, "completions/min_terminated_length": 659.0, "entropy": 0.6663954973220825, "epoch": 0.012001200120012, "frac_reward_zero_std": 0.0, "grad_norm": 1.2734375, "learning_rate": 8.105515587529975e-07, "loss": 0.0095, "num_tokens": 8922980.0, "reward": 0.29745977818965913, "reward_std": 0.28679852187633514, "rewards/ADEnReward/mean": 0.03675286723300815, "rewards/ADEnReward/std": 0.10012084320187568, "rewards/CriticV2Reward/mean": 0.16487355828285216, "rewards/CriticV2Reward/std": 0.2441360980272293, "rewards/StrictFormatReward/mean": 0.9583333253860473, "rewards/StrictFormatReward/std": 0.21331926584243774, "sampling/importance_sampling_ratio/max": 2.6717817783355713, "sampling/importance_sampling_ratio/mean": 0.40106405019760133, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.966424179077149, "sampling/sampling_logp_difference/mean": 0.0272501390427351, "step": 80, "step_time": 13.397188885137439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 824.0, "completions/max_terminated_length": 824.0, "completions/mean_length": 728.7416870117188, "completions/mean_terminated_length": 728.7416870117188, "completions/min_length": 578.0, "completions/min_terminated_length": 578.0, "entropy": 0.6762633740901947, "epoch": 0.012751275127512752, "frac_reward_zero_std": 0.0, "grad_norm": 0.83203125, "learning_rate": 7.985611510791367e-07, "loss": -0.0382, "num_tokens": 9478630.0, "reward": 0.29869637489318845, "reward_std": 0.2758790373802185, "rewards/ADEnReward/mean": 0.021353329718112945, "rewards/ADEnReward/std": 0.052459438890218736, "rewards/CriticV2Reward/mean": 0.17984303086996078, "rewards/CriticV2Reward/std": 0.2615115612745285, "rewards/StrictFormatReward/mean": 0.975, "rewards/StrictFormatReward/std": 0.13851140141487123, "sampling/importance_sampling_ratio/max": 2.4585710525512696, "sampling/importance_sampling_ratio/mean": 0.36855998635292053, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.735863780975342, "sampling/sampling_logp_difference/mean": 0.027072810009121893, "step": 85, "step_time": 13.575490239448845 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 945.0, "completions/max_terminated_length": 945.0, "completions/mean_length": 728.6083618164063, "completions/mean_terminated_length": 728.6083618164063, "completions/min_length": 535.6, "completions/min_terminated_length": 535.6, "entropy": 0.6834039151668548, "epoch": 0.013501350135013501, "frac_reward_zero_std": 0.0, "grad_norm": 0.421875, "learning_rate": 7.865707434052757e-07, "loss": -0.0357, "num_tokens": 10034216.0, "reward": 0.25413125455379487, "reward_std": 0.23914098739624023, "rewards/ADEnReward/mean": 0.04056290723383427, "rewards/ADEnReward/std": 0.09261687994003295, "rewards/CriticV2Reward/mean": 0.12273500114679337, "rewards/CriticV2Reward/std": 0.2090004175901413, "rewards/StrictFormatReward/mean": 0.9083333373069763, "rewards/StrictFormatReward/std": 0.31918213367462156, "sampling/importance_sampling_ratio/max": 2.5320541858673096, "sampling/importance_sampling_ratio/mean": 0.4195625066757202, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6738919258117675, "sampling/sampling_logp_difference/mean": 0.02621118016541004, "step": 90, "step_time": 14.163981208670885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1055.0, "completions/max_terminated_length": 1055.0, "completions/mean_length": 734.9208618164063, "completions/mean_terminated_length": 734.9208618164063, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.6778877079486847, "epoch": 0.014251425142514252, "frac_reward_zero_std": 0.0, "grad_norm": 0.54296875, "learning_rate": 7.745803357314147e-07, "loss": 0.0453, "num_tokens": 10591749.0, "reward": 0.2550377070903778, "reward_std": 0.24310122430324554, "rewards/ADEnReward/mean": 0.011902040056884288, "rewards/ADEnReward/std": 0.032011125981807706, "rewards/CriticV2Reward/mean": 0.14980231374502181, "rewards/CriticV2Reward/std": 0.2343314677476883, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3577991783618927, "sampling/importance_sampling_ratio/max": 2.2254454612731935, "sampling/importance_sampling_ratio/mean": 0.33642452359199526, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.6567583084106445, "sampling/sampling_logp_difference/mean": 0.02788180038332939, "step": 95, "step_time": 15.019571543950587 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 805.8, "completions/max_terminated_length": 805.8, "completions/mean_length": 724.3875244140625, "completions/mean_terminated_length": 724.3875244140625, "completions/min_length": 638.0, "completions/min_terminated_length": 638.0, "entropy": 0.6795200049877167, "epoch": 0.015001500150015001, "frac_reward_zero_std": 0.0, "grad_norm": 0.458984375, "learning_rate": 7.62589928057554e-07, "loss": -0.0563, "num_tokens": 11145906.0, "reward": 0.27770817279815674, "reward_std": 0.2380681112408638, "rewards/ADEnReward/mean": 0.048516617715358736, "rewards/ADEnReward/std": 0.1014238491654396, "rewards/CriticV2Reward/mean": 0.13669154196977615, "rewards/CriticV2Reward/std": 0.20003681480884553, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.36570553183555604, "sampling/importance_sampling_ratio/max": 2.1466657400131224, "sampling/importance_sampling_ratio/mean": 0.3968630850315094, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.029999828338623, "sampling/sampling_logp_difference/mean": 0.026145283505320548, "step": 100, "step_time": 13.395550060458481 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 816.4, "completions/max_terminated_length": 816.4, "completions/mean_length": 735.120849609375, "completions/mean_terminated_length": 735.120849609375, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "entropy": 0.6767585337162018, "epoch": 0.01575157515751575, "frac_reward_zero_std": 0.0, "grad_norm": 0.5078125, "learning_rate": 7.50599520383693e-07, "loss": -0.0046, "num_tokens": 11703711.0, "reward": 0.22518396079540254, "reward_std": 0.21424060463905334, "rewards/ADEnReward/mean": 0.02892113085836172, "rewards/ADEnReward/std": 0.07816565409302711, "rewards/CriticV2Reward/mean": 0.10376282632350922, "rewards/CriticV2Reward/std": 0.1866258502006531, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.36890350580215453, "sampling/importance_sampling_ratio/max": 2.333189105987549, "sampling/importance_sampling_ratio/mean": 0.3842966675758362, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3249330043792726, "sampling/sampling_logp_difference/mean": 0.027384335547685622, "step": 105, "step_time": 13.222316846810282 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 812.2, "completions/max_terminated_length": 812.2, "completions/mean_length": 727.433349609375, "completions/mean_terminated_length": 727.433349609375, "completions/min_length": 646.4, "completions/min_terminated_length": 646.4, "entropy": 0.6769951701164245, "epoch": 0.0165016501650165, "frac_reward_zero_std": 0.0, "grad_norm": 0.53515625, "learning_rate": 7.386091127098321e-07, "loss": -0.0397, "num_tokens": 12258999.0, "reward": 0.23659175038337707, "reward_std": 0.2429301619529724, "rewards/ADEnReward/mean": 0.02100809020921588, "rewards/ADEnReward/std": 0.06905662044882774, "rewards/CriticV2Reward/mean": 0.12225031852722168, "rewards/CriticV2Reward/std": 0.21501878201961516, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3171369910240173, "sampling/importance_sampling_ratio/max": 2.3933506488800047, "sampling/importance_sampling_ratio/mean": 0.34748960137367246, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1887176513671873, "sampling/sampling_logp_difference/mean": 0.02707708366215229, "step": 110, "step_time": 13.317681908141822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1086.4, "completions/max_terminated_length": 945.4, "completions/mean_length": 739.9375244140625, "completions/mean_terminated_length": 736.5984985351563, "completions/min_length": 557.4, "completions/min_terminated_length": 557.4, "entropy": 0.6870240211486817, "epoch": 0.017251725172517253, "frac_reward_zero_std": 0.0, "grad_norm": 0.330078125, "learning_rate": 7.266187050359712e-07, "loss": 0.0218, "num_tokens": 12817736.0, "reward": 0.25257408022880556, "reward_std": 0.2376851588487625, "rewards/ADEnReward/mean": 0.027911533229053022, "rewards/ADEnReward/std": 0.07045800052583218, "rewards/CriticV2Reward/mean": 0.13216253668069838, "rewards/CriticV2Reward/std": 0.21919724345207214, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.37487201690673827, "sampling/importance_sampling_ratio/max": 2.0126373291015627, "sampling/importance_sampling_ratio/mean": 0.35888453125953673, "sampling/importance_sampling_ratio/min": 5.0439726328477265e-05, "sampling/sampling_logp_difference/max": 3.8658429384231567, "sampling/sampling_logp_difference/mean": 0.02792995758354664, "step": 115, "step_time": 14.98624010598287 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 813.2, "completions/max_terminated_length": 813.2, "completions/mean_length": 727.6916870117187, "completions/mean_terminated_length": 727.6916870117187, "completions/min_length": 538.4, "completions/min_terminated_length": 538.4, "entropy": 0.6622766137123108, "epoch": 0.018001800180018002, "frac_reward_zero_std": 0.0, "grad_norm": 0.4453125, "learning_rate": 7.146282973621102e-07, "loss": -0.0632, "num_tokens": 13373374.0, "reward": 0.2659685522317886, "reward_std": 0.2437886893749237, "rewards/ADEnReward/mean": 0.023663881793618204, "rewards/ADEnReward/std": 0.05939733125269413, "rewards/CriticV2Reward/mean": 0.14813800007104874, "rewards/CriticV2Reward/std": 0.21836518943309785, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.30006415247917173, "sampling/importance_sampling_ratio/max": 2.6608290672302246, "sampling/importance_sampling_ratio/mean": 0.40939287543296815, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.3562741041183473, "sampling/sampling_logp_difference/mean": 0.027171416208148003, "step": 120, "step_time": 13.383991296868771 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1089.8, "completions/max_terminated_length": 798.4, "completions/mean_length": 738.9291870117188, "completions/mean_terminated_length": 732.2929931640625, "completions/min_length": 654.0, "completions/min_terminated_length": 654.0, "entropy": 0.6830495119094848, "epoch": 0.01875187518751875, "frac_reward_zero_std": 0.0, "grad_norm": 0.478515625, "learning_rate": 7.026378896882493e-07, "loss": 0.0473, "num_tokens": 13931613.0, "reward": 0.26927419006824493, "reward_std": 0.24791643023490906, "rewards/ADEnReward/mean": 0.029334013583138585, "rewards/ADEnReward/std": 0.06973049007356166, "rewards/CriticV2Reward/mean": 0.1432734966278076, "rewards/CriticV2Reward/std": 0.2263641268014908, "rewards/StrictFormatReward/mean": 0.9666666507720947, "rewards/StrictFormatReward/std": 0.2309401035308838, "sampling/importance_sampling_ratio/max": 2.6443384647369386, "sampling/importance_sampling_ratio/mean": 0.44446309208869933, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.7472902774810795, "sampling/sampling_logp_difference/mean": 0.027243128046393394, "step": 125, "step_time": 15.178255619574339 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 964.0, "completions/max_terminated_length": 816.8, "completions/mean_length": 735.46669921875, "completions/mean_terminated_length": 732.1085327148437, "completions/min_length": 608.0, "completions/min_terminated_length": 608.0, "entropy": 0.6759671211242676, "epoch": 0.0195019501950195, "frac_reward_zero_std": 0.0, "grad_norm": 0.48046875, "learning_rate": 6.906474820143885e-07, "loss": 0.025, "num_tokens": 14489661.0, "reward": 0.31296299397945404, "reward_std": 0.2914800405502319, "rewards/ADEnReward/mean": 0.034117148048244415, "rewards/ADEnReward/std": 0.08479407420381904, "rewards/CriticV2Reward/mean": 0.18801250755786897, "rewards/CriticV2Reward/std": 0.26688812375068666, "rewards/StrictFormatReward/mean": 0.9083333373069763, "rewards/StrictFormatReward/std": 0.3529752492904663, "sampling/importance_sampling_ratio/max": 2.6238924980163576, "sampling/importance_sampling_ratio/mean": 0.36424337029457093, "sampling/importance_sampling_ratio/min": 9.081272291950881e-05, "sampling/sampling_logp_difference/max": 2.2420372009277343, "sampling/sampling_logp_difference/mean": 0.027383123710751533, "step": 130, "step_time": 14.45475140977651 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 817.4, "completions/max_terminated_length": 817.4, "completions/mean_length": 730.6375244140625, "completions/mean_terminated_length": 730.6375244140625, "completions/min_length": 547.2, "completions/min_terminated_length": 547.2, "entropy": 0.6457026600837708, "epoch": 0.02025202520252025, "frac_reward_zero_std": 0.0, "grad_norm": 0.7265625, "learning_rate": 6.786570743405276e-07, "loss": -0.0069, "num_tokens": 15046086.0, "reward": 0.30475842356681826, "reward_std": 0.29704451858997344, "rewards/ADEnReward/mean": 0.029951873375102876, "rewards/ADEnReward/std": 0.0687396578490734, "rewards/CriticV2Reward/mean": 0.18313985466957092, "rewards/CriticV2Reward/std": 0.2745169162750244, "rewards/StrictFormatReward/mean": 0.9166666746139527, "rewards/StrictFormatReward/std": 0.30210930705070493, "sampling/importance_sampling_ratio/max": 2.5814715385437013, "sampling/importance_sampling_ratio/mean": 0.4488029360771179, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.662810564041138, "sampling/sampling_logp_difference/mean": 0.026542375236749648, "step": 135, "step_time": 13.331754672527314 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 802.8, "completions/max_terminated_length": 802.8, "completions/mean_length": 734.3500122070312, "completions/mean_terminated_length": 734.3500122070312, "completions/min_length": 667.2, "completions/min_terminated_length": 667.2, "entropy": 0.6539111316204071, "epoch": 0.021002100210021003, "frac_reward_zero_std": 0.0, "grad_norm": 0.2578125, "learning_rate": 6.666666666666666e-07, "loss": 0.0054, "num_tokens": 15603610.0, "reward": 0.32718279361724856, "reward_std": 0.2756353974342346, "rewards/ADEnReward/mean": 0.035100750438869, "rewards/ADEnReward/std": 0.0759646624326706, "rewards/CriticV2Reward/mean": 0.19708201587200164, "rewards/CriticV2Reward/std": 0.2542935281991959, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.27702280282974245, "sampling/importance_sampling_ratio/max": 1.9317828178405763, "sampling/importance_sampling_ratio/mean": 0.2619578391313553, "sampling/importance_sampling_ratio/min": 0.0020720528438687323, "sampling/sampling_logp_difference/max": 3.8336150884628295, "sampling/sampling_logp_difference/mean": 0.02702697478234768, "step": 140, "step_time": 13.271004169993102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 864.4, "completions/max_terminated_length": 864.4, "completions/mean_length": 728.5625, "completions/mean_terminated_length": 728.5625, "completions/min_length": 560.4, "completions/min_terminated_length": 560.4, "entropy": 0.6553530871868134, "epoch": 0.021752175217521753, "frac_reward_zero_std": 0.0, "grad_norm": 0.2890625, "learning_rate": 6.546762589928057e-07, "loss": -0.0144, "num_tokens": 16159489.0, "reward": 0.3294836848974228, "reward_std": 0.3109097331762314, "rewards/ADEnReward/mean": 0.03947688639163971, "rewards/ADEnReward/std": 0.08423196449875832, "rewards/CriticV2Reward/mean": 0.19584011435508727, "rewards/CriticV2Reward/std": 0.2782162755727768, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.25343345403671264, "sampling/importance_sampling_ratio/max": 2.2559653520584106, "sampling/importance_sampling_ratio/mean": 0.39474677443504336, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.220552110671997, "sampling/sampling_logp_difference/mean": 0.026592973992228507, "step": 145, "step_time": 13.792286728322505 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 816.2, "completions/max_terminated_length": 816.2, "completions/mean_length": 727.458349609375, "completions/mean_terminated_length": 727.458349609375, "completions/min_length": 528.8, "completions/min_terminated_length": 528.8, "entropy": 0.6875123620033264, "epoch": 0.022502250225022502, "frac_reward_zero_std": 0.0, "grad_norm": 0.82421875, "learning_rate": 6.426858513189448e-07, "loss": 0.0156, "num_tokens": 16715023.0, "reward": 0.2460826128721237, "reward_std": 0.2182670548558235, "rewards/ADEnReward/mean": 0.021830105502158404, "rewards/ADEnReward/std": 0.05803550221025944, "rewards/CriticV2Reward/mean": 0.1309191606938839, "rewards/CriticV2Reward/std": 0.20253435969352723, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.30797049403190613, "sampling/importance_sampling_ratio/max": 2.603206253051758, "sampling/importance_sampling_ratio/mean": 0.3743092954158783, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.419530653953552, "sampling/sampling_logp_difference/mean": 0.027796153724193574, "step": 150, "step_time": 13.406930388044566 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1140.0, "completions/max_terminated_length": 995.0, "completions/mean_length": 746.908349609375, "completions/mean_terminated_length": 743.5602905273438, "completions/min_length": 667.6, "completions/min_terminated_length": 667.6, "entropy": 0.6762481391429901, "epoch": 0.02325232523252325, "frac_reward_zero_std": 0.0, "grad_norm": 0.4609375, "learning_rate": 6.306954436450838e-07, "loss": -0.027, "num_tokens": 17275209.0, "reward": 0.319897449016571, "reward_std": 0.2809853881597519, "rewards/ADEnReward/mean": 0.046538510359823704, "rewards/ADEnReward/std": 0.09964709505438804, "rewards/CriticV2Reward/mean": 0.1800255909562111, "rewards/CriticV2Reward/std": 0.2593708336353302, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.3111684799194336, "sampling/importance_sampling_ratio/max": 1.9672291159629822, "sampling/importance_sampling_ratio/mean": 0.37010484337806704, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.342987585067749, "sampling/sampling_logp_difference/mean": 0.027148356661200523, "step": 155, "step_time": 15.492892314493655 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 847.4, "completions/max_terminated_length": 847.4, "completions/mean_length": 730.895849609375, "completions/mean_terminated_length": 730.895849609375, "completions/min_length": 670.2, "completions/min_terminated_length": 670.2, "entropy": 0.6781861484050751, "epoch": 0.024002400240024, "frac_reward_zero_std": 0.0, "grad_norm": 0.8359375, "learning_rate": 6.187050359712231e-07, "loss": -0.0039, "num_tokens": 17831280.0, "reward": 0.2673605889081955, "reward_std": 0.2609496355056763, "rewards/ADEnReward/mean": 0.022430585324764253, "rewards/ADEnReward/std": 0.05508136935532093, "rewards/CriticV2Reward/mean": 0.14826332926750183, "rewards/CriticV2Reward/std": 0.23569288849830627, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.6008774757385256, "sampling/importance_sampling_ratio/mean": 0.34396409392356875, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0486708641052247, "sampling/sampling_logp_difference/mean": 0.0269806656986475, "step": 160, "step_time": 13.56647696653381 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 803.2, "completions/max_terminated_length": 803.2, "completions/mean_length": 723.6166748046875, "completions/mean_terminated_length": 723.6166748046875, "completions/min_length": 542.0, "completions/min_terminated_length": 542.0, "entropy": 0.6606142938137054, "epoch": 0.024752475247524754, "frac_reward_zero_std": 0.0, "grad_norm": 0.9453125, "learning_rate": 6.067146282973621e-07, "loss": -0.0583, "num_tokens": 18385348.0, "reward": 0.2865483522415161, "reward_std": 0.2430926740169525, "rewards/ADEnReward/mean": 0.038159405998885634, "rewards/ADEnReward/std": 0.09702088348567486, "rewards/CriticV2Reward/mean": 0.15088892579078675, "rewards/CriticV2Reward/std": 0.22967694103717803, "rewards/StrictFormatReward/mean": 0.9749999880790711, "rewards/StrictFormatReward/std": 0.17320507764816284, "sampling/importance_sampling_ratio/max": 2.3871933221817017, "sampling/importance_sampling_ratio/mean": 0.3935186803340912, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7643911838531494, "sampling/sampling_logp_difference/mean": 0.02641451619565487, "step": 165, "step_time": 13.218168735411018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 952.4, "completions/max_terminated_length": 808.0, "completions/mean_length": 730.1541870117187, "completions/mean_terminated_length": 726.7688110351562, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.6817957520484924, "epoch": 0.025502550255025503, "frac_reward_zero_std": 0.0, "grad_norm": 0.466796875, "learning_rate": 5.947242206235011e-07, "loss": -0.11, "num_tokens": 18941433.0, "reward": 0.2676409512758255, "reward_std": 0.24391051530838012, "rewards/ADEnReward/mean": 0.03495770245790482, "rewards/ADEnReward/std": 0.08008336126804352, "rewards/CriticV2Reward/mean": 0.14184990525245667, "rewards/CriticV2Reward/std": 0.2124081790447235, "rewards/StrictFormatReward/mean": 0.9083333373069763, "rewards/StrictFormatReward/std": 0.4005065083503723, "sampling/importance_sampling_ratio/max": 2.514456629753113, "sampling/importance_sampling_ratio/mean": 0.3926523685455322, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.343113994598388, "sampling/sampling_logp_difference/mean": 0.027290407940745352, "step": 170, "step_time": 14.132117554545403 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 952.8, "completions/max_terminated_length": 952.8, "completions/mean_length": 734.9291748046875, "completions/mean_terminated_length": 734.9291748046875, "completions/min_length": 572.0, "completions/min_terminated_length": 572.0, "entropy": 0.6970304548740387, "epoch": 0.026252625262526252, "frac_reward_zero_std": 0.0, "grad_norm": 0.69921875, "learning_rate": 5.827338129496403e-07, "loss": 0.0331, "num_tokens": 19498808.0, "reward": 0.27273200154304506, "reward_std": 0.2695725470781326, "rewards/ADEnReward/mean": 0.01761272009462118, "rewards/ADEnReward/std": 0.06024995818734169, "rewards/CriticV2Reward/mean": 0.1609526053071022, "rewards/CriticV2Reward/std": 0.25695641338825226, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.25343345403671264, "sampling/importance_sampling_ratio/max": 2.529092001914978, "sampling/importance_sampling_ratio/mean": 0.39281960129737853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.5247619390487674, "sampling/sampling_logp_difference/mean": 0.026818586885929106, "step": 175, "step_time": 14.21546617038548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 831.6, "completions/max_terminated_length": 831.6, "completions/mean_length": 732.2125122070313, "completions/mean_terminated_length": 732.2125122070313, "completions/min_length": 675.6, "completions/min_terminated_length": 675.6, "entropy": 0.6531910359859466, "epoch": 0.027002700270027002, "frac_reward_zero_std": 0.0, "grad_norm": 0.484375, "learning_rate": 5.707434052757793e-07, "loss": -0.0202, "num_tokens": 20055611.0, "reward": 0.314579039812088, "reward_std": 0.2669638603925705, "rewards/ADEnReward/mean": 0.050490521267056464, "rewards/ADEnReward/std": 0.09671776592731476, "rewards/CriticV2Reward/mean": 0.1657551869750023, "rewards/CriticV2Reward/std": 0.2505346149206161, "rewards/StrictFormatReward/mean": 0.9833333373069764, "rewards/StrictFormatReward/std": 0.08077637553215027, "sampling/importance_sampling_ratio/max": 2.2050620555877685, "sampling/importance_sampling_ratio/mean": 0.3497112452983856, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.723379349708557, "sampling/sampling_logp_difference/mean": 0.026673595979809762, "step": 180, "step_time": 13.579954623430968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1110.2, "completions/max_terminated_length": 821.6, "completions/mean_length": 738.7416870117188, "completions/mean_terminated_length": 731.9643676757812, "completions/min_length": 656.6, "completions/min_terminated_length": 656.6, "entropy": 0.6632063090801239, "epoch": 0.02775277527752775, "frac_reward_zero_std": 0.0, "grad_norm": 0.58203125, "learning_rate": 5.587529976019184e-07, "loss": -0.0011, "num_tokens": 20614061.0, "reward": 0.2985087275505066, "reward_std": 0.2817197978496552, "rewards/ADEnReward/mean": 0.04548268616199493, "rewards/ADEnReward/std": 0.10240544080734253, "rewards/CriticV2Reward/mean": 0.16135935932397844, "rewards/CriticV2Reward/std": 0.24745513498783112, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.29294281005859374, "sampling/importance_sampling_ratio/max": 2.2299548625946044, "sampling/importance_sampling_ratio/mean": 0.36481807231903074, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.58858847618103, "sampling/sampling_logp_difference/mean": 0.027159354835748672, "step": 185, "step_time": 15.367878540046513 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 817.8, "completions/max_terminated_length": 817.8, "completions/mean_length": 732.2916870117188, "completions/mean_terminated_length": 732.2916870117188, "completions/min_length": 536.8, "completions/min_terminated_length": 536.8, "entropy": 0.6737968623638153, "epoch": 0.028502850285028504, "frac_reward_zero_std": 0.0, "grad_norm": 0.67578125, "learning_rate": 5.467625899280576e-07, "loss": -0.017, "num_tokens": 21170995.0, "reward": 0.2543010950088501, "reward_std": 0.237994921207428, "rewards/ADEnReward/mean": 0.03096686936914921, "rewards/ADEnReward/std": 0.07621146515011787, "rewards/CriticV2Reward/mean": 0.13000087440013885, "rewards/CriticV2Reward/std": 0.2169148951768875, "rewards/StrictFormatReward/mean": 0.9333333253860474, "rewards/StrictFormatReward/std": 0.35183066725730894, "sampling/importance_sampling_ratio/max": 2.219161319732666, "sampling/importance_sampling_ratio/mean": 0.3931430697441101, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.842717552185059, "sampling/sampling_logp_difference/mean": 0.0267446331679821, "step": 190, "step_time": 13.365200480632485 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 806.2, "completions/max_terminated_length": 806.2, "completions/mean_length": 726.0291870117187, "completions/mean_terminated_length": 726.0291870117187, "completions/min_length": 591.6, "completions/min_terminated_length": 591.6, "entropy": 0.6693301379680634, "epoch": 0.029252925292529253, "frac_reward_zero_std": 0.0, "grad_norm": 0.703125, "learning_rate": 5.347721822541966e-07, "loss": -0.0212, "num_tokens": 21725930.0, "reward": 0.2516060918569565, "reward_std": 0.22183270156383514, "rewards/ADEnReward/mean": 0.030396451149135827, "rewards/ADEnReward/std": 0.07066891826689244, "rewards/CriticV2Reward/mean": 0.128709626942873, "rewards/CriticV2Reward/std": 0.20975570380687714, "rewards/StrictFormatReward/mean": 0.925000011920929, "rewards/StrictFormatReward/std": 0.380840528011322, "sampling/importance_sampling_ratio/max": 2.3974624156951903, "sampling/importance_sampling_ratio/mean": 0.35138731002807616, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.941667985916138, "sampling/sampling_logp_difference/mean": 0.026314648613333703, "step": 195, "step_time": 13.368891882430763 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 954.2, "completions/max_terminated_length": 954.2, "completions/mean_length": 736.1416748046875, "completions/mean_terminated_length": 736.1416748046875, "completions/min_length": 668.0, "completions/min_terminated_length": 668.0, "entropy": 0.6977671384811401, "epoch": 0.030003000300030003, "frac_reward_zero_std": 0.0, "grad_norm": 0.60546875, "learning_rate": 5.227817745803357e-07, "loss": 0.0251, "num_tokens": 22283580.0, "reward": 0.25325133800506594, "reward_std": 0.23282511830329894, "rewards/ADEnReward/mean": 0.013752670167014003, "rewards/ADEnReward/std": 0.03788870992138982, "rewards/CriticV2Reward/mean": 0.14449864774942398, "rewards/CriticV2Reward/std": 0.22673717141151428, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.27702280282974245, "sampling/importance_sampling_ratio/max": 1.8331253051757812, "sampling/importance_sampling_ratio/mean": 0.32110574245452883, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 6.006078147888184, "sampling/sampling_logp_difference/mean": 0.027469008043408395, "step": 200, "step_time": 14.366346049588174 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 814.6, "completions/max_terminated_length": 814.6, "completions/mean_length": 729.6416870117188, "completions/mean_terminated_length": 729.6416870117188, "completions/min_length": 548.8, "completions/min_terminated_length": 548.8, "entropy": 0.6385031104087829, "epoch": 0.030753075307530752, "frac_reward_zero_std": 0.0, "grad_norm": 0.83203125, "learning_rate": 5.107913669064748e-07, "loss": -0.0242, "num_tokens": 22839654.0, "reward": 0.3597941994667053, "reward_std": 0.32995885014534, "rewards/ADEnReward/mean": 0.0433755524456501, "rewards/ADEnReward/std": 0.0839846022427082, "rewards/CriticV2Reward/mean": 0.22225197553634643, "rewards/CriticV2Reward/std": 0.2984564691781998, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.3347578287124634, "sampling/importance_sampling_ratio/max": 2.4754393100738525, "sampling/importance_sampling_ratio/mean": 0.4427867650985718, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.708860540390015, "sampling/sampling_logp_difference/mean": 0.02616884931921959, "step": 205, "step_time": 13.387136839143931 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 921.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 729.7416870117188, "completions/mean_terminated_length": 729.7416870117188, "completions/min_length": 545.2, "completions/min_terminated_length": 545.2, "entropy": 0.6550398588180542, "epoch": 0.0315031503150315, "frac_reward_zero_std": 0.0, "grad_norm": 0.66796875, "learning_rate": 4.988009592326139e-07, "loss": 0.0169, "num_tokens": 23395880.0, "reward": 0.26131516993045806, "reward_std": 0.2235265478491783, "rewards/ADEnReward/mean": 0.015563993342220783, "rewards/ADEnReward/std": 0.04172728545963764, "rewards/CriticV2Reward/mean": 0.15325116813182832, "rewards/CriticV2Reward/std": 0.22710732668638228, "rewards/StrictFormatReward/mean": 0.924999988079071, "rewards/StrictFormatReward/std": 0.3657055199146271, "sampling/importance_sampling_ratio/max": 2.1074886083602906, "sampling/importance_sampling_ratio/mean": 0.317373925447464, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.4951153755187985, "sampling/sampling_logp_difference/mean": 0.026737869903445243, "step": 210, "step_time": 14.018542888667435 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 829.2, "completions/max_terminated_length": 829.2, "completions/mean_length": 737.1625244140625, "completions/mean_terminated_length": 737.1625244140625, "completions/min_length": 663.2, "completions/min_terminated_length": 663.2, "entropy": 0.6886229276657104, "epoch": 0.032253225322532254, "frac_reward_zero_std": 0.0, "grad_norm": 0.609375, "learning_rate": 4.86810551558753e-07, "loss": -0.0492, "num_tokens": 23954047.0, "reward": 0.28127689361572267, "reward_std": 0.24524502754211425, "rewards/ADEnReward/mean": 0.030380566231906415, "rewards/ADEnReward/std": 0.06638182587921619, "rewards/CriticV2Reward/mean": 0.15589632242918014, "rewards/CriticV2Reward/std": 0.22414239943027497, "rewards/StrictFormatReward/mean": 0.950000011920929, "rewards/StrictFormatReward/std": 0.2423291265964508, "sampling/importance_sampling_ratio/max": 2.2944180488586428, "sampling/importance_sampling_ratio/mean": 0.35170907974243165, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.074922227859497, "sampling/sampling_logp_difference/mean": 0.02738656513392925, "step": 215, "step_time": 13.396183334942908 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 918.8, "completions/max_terminated_length": 918.8, "completions/mean_length": 733.0500122070313, "completions/mean_terminated_length": 733.0500122070313, "completions/min_length": 664.4, "completions/min_terminated_length": 664.4, "entropy": 0.6667313754558564, "epoch": 0.033003300330033, "frac_reward_zero_std": 0.0, "grad_norm": 0.466796875, "learning_rate": 4.7482014388489204e-07, "loss": 0.0128, "num_tokens": 24511323.0, "reward": 0.27115337550640106, "reward_std": 0.25859386622905733, "rewards/ADEnReward/mean": 0.02348469356074929, "rewards/ADEnReward/std": 0.06776928193867207, "rewards/CriticV2Reward/mean": 0.15766866207122804, "rewards/CriticV2Reward/std": 0.2331855744123459, "rewards/StrictFormatReward/mean": 0.9, "rewards/StrictFormatReward/std": 0.3700480878353119, "sampling/importance_sampling_ratio/max": 2.167785573005676, "sampling/importance_sampling_ratio/mean": 0.34185417294502257, "sampling/importance_sampling_ratio/min": 2.1186770754866303e-05, "sampling/sampling_logp_difference/max": 5.053237056732177, "sampling/sampling_logp_difference/mean": 0.027166034281253814, "step": 220, "step_time": 14.092069184035063 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 793.2, "completions/max_terminated_length": 793.2, "completions/mean_length": 727.0125244140625, "completions/mean_terminated_length": 727.0125244140625, "completions/min_length": 543.4, "completions/min_terminated_length": 543.4, "entropy": 0.6576771378517151, "epoch": 0.03375337533753375, "frac_reward_zero_std": 0.0, "grad_norm": 0.421875, "learning_rate": 4.628297362110312e-07, "loss": -0.0292, "num_tokens": 25066702.0, "reward": 0.28278968334197996, "reward_std": 0.277101942896843, "rewards/ADEnReward/mean": 0.04402967318892479, "rewards/ADEnReward/std": 0.10327454507350922, "rewards/CriticV2Reward/mean": 0.14209333211183547, "rewards/CriticV2Reward/std": 0.22946595251560212, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.48654568195343, "sampling/importance_sampling_ratio/mean": 0.37147427201271055, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.846973514556885, "sampling/sampling_logp_difference/mean": 0.026473153755068778, "step": 225, "step_time": 13.257498236745596 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 921.4, "completions/max_terminated_length": 921.4, "completions/mean_length": 733.683349609375, "completions/mean_terminated_length": 733.683349609375, "completions/min_length": 661.6, "completions/min_terminated_length": 661.6, "entropy": 0.6625209391117096, "epoch": 0.034503450345034506, "frac_reward_zero_std": 0.0, "grad_norm": 0.5546875, "learning_rate": 4.508393285371702e-07, "loss": -0.0321, "num_tokens": 25624018.0, "reward": 0.310102254152298, "reward_std": 0.29136236310005187, "rewards/ADEnReward/mean": 0.029331159964203834, "rewards/ADEnReward/std": 0.07541338577866555, "rewards/CriticV2Reward/mean": 0.18577107340097426, "rewards/CriticV2Reward/std": 0.26120834350585936, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.23636061549186707, "sampling/importance_sampling_ratio/max": 2.2946805477142336, "sampling/importance_sampling_ratio/mean": 0.36944074034690855, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.595243453979492, "sampling/sampling_logp_difference/mean": 0.026906453445553778, "step": 230, "step_time": 14.133570789452643 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1078.8, "completions/max_terminated_length": 1055.0, "completions/mean_length": 737.0583618164062, "completions/mean_terminated_length": 733.7565795898438, "completions/min_length": 561.8, "completions/min_terminated_length": 561.8, "entropy": 0.6722292423248291, "epoch": 0.03525352535253525, "frac_reward_zero_std": 0.0, "grad_norm": 0.478515625, "learning_rate": 4.388489208633093e-07, "loss": 0.0154, "num_tokens": 26181936.0, "reward": 0.300505068898201, "reward_std": 0.28410873413085935, "rewards/ADEnReward/mean": 0.030564435198903082, "rewards/ADEnReward/std": 0.09145040065050125, "rewards/CriticV2Reward/mean": 0.178273943066597, "rewards/CriticV2Reward/std": 0.26288277208805083, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.34808468222618105, "sampling/importance_sampling_ratio/max": 2.5372284412384034, "sampling/importance_sampling_ratio/mean": 0.3265813708305359, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.361432075500488, "sampling/sampling_logp_difference/mean": 0.02705967016518116, "step": 235, "step_time": 14.942696164734661 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 950.8, "completions/max_terminated_length": 869.6, "completions/mean_length": 734.570849609375, "completions/mean_terminated_length": 731.2459228515625, "completions/min_length": 654.4, "completions/min_terminated_length": 654.4, "entropy": 0.665759927034378, "epoch": 0.036003600360036005, "frac_reward_zero_std": 0.0, "grad_norm": 0.5859375, "learning_rate": 4.2685851318944845e-07, "loss": -0.0562, "num_tokens": 26739273.0, "reward": 0.31318188905715943, "reward_std": 0.2850573241710663, "rewards/ADEnReward/mean": 0.02819870375096798, "rewards/ADEnReward/std": 0.0679096095263958, "rewards/CriticV2Reward/mean": 0.19248318374156953, "rewards/CriticV2Reward/std": 0.27355403900146485, "rewards/StrictFormatReward/mean": 0.925000011920929, "rewards/StrictFormatReward/std": 0.28503645658493043, "sampling/importance_sampling_ratio/max": 2.6505446910858153, "sampling/importance_sampling_ratio/mean": 0.3801407516002655, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.5118245124816894, "sampling/sampling_logp_difference/mean": 0.02762332819402218, "step": 240, "step_time": 14.368941375706346 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 948.4, "completions/max_terminated_length": 798.4, "completions/mean_length": 732.2875122070312, "completions/mean_terminated_length": 728.9180908203125, "completions/min_length": 651.6, "completions/min_terminated_length": 651.6, "entropy": 0.6761650085449219, "epoch": 0.03675367536753675, "frac_reward_zero_std": 0.0, "grad_norm": 0.66015625, "learning_rate": 4.148681055155875e-07, "loss": 0.0613, "num_tokens": 27295806.0, "reward": 0.2632507473230362, "reward_std": 0.23623428642749786, "rewards/ADEnReward/mean": 0.028980655129998922, "rewards/ADEnReward/std": 0.06641366388648748, "rewards/CriticV2Reward/mean": 0.14260342791676522, "rewards/CriticV2Reward/std": 0.2100657716393471, "rewards/StrictFormatReward/mean": 0.9166666746139527, "rewards/StrictFormatReward/std": 0.3979133665561676, "sampling/importance_sampling_ratio/max": 2.3929349422454833, "sampling/importance_sampling_ratio/mean": 0.4016247749328613, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.0486182451248167, "sampling/sampling_logp_difference/mean": 0.026698941737413405, "step": 245, "step_time": 14.183489987812937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 943.8, "completions/max_terminated_length": 943.8, "completions/mean_length": 733.6750366210938, "completions/mean_terminated_length": 733.6750366210938, "completions/min_length": 661.6, "completions/min_terminated_length": 661.6, "entropy": 0.6711142897605896, "epoch": 0.0375037503750375, "frac_reward_zero_std": 0.0, "grad_norm": 0.6328125, "learning_rate": 4.028776978417266e-07, "loss": -0.0507, "num_tokens": 27852816.0, "reward": 0.2927826702594757, "reward_std": 0.2535986423492432, "rewards/ADEnReward/mean": 0.0256393326446414, "rewards/ADEnReward/std": 0.059096282720565795, "rewards/CriticV2Reward/mean": 0.17380999326705932, "rewards/CriticV2Reward/std": 0.2408513218164444, "rewards/StrictFormatReward/mean": 0.9333333373069763, "rewards/StrictFormatReward/std": 0.3577991783618927, "sampling/importance_sampling_ratio/max": 2.528782033920288, "sampling/importance_sampling_ratio/mean": 0.38217838406562804, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.319740128517151, "sampling/sampling_logp_difference/mean": 0.026942050084471703, "step": 250, "step_time": 14.233225086890162 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 978.6, "completions/max_terminated_length": 978.6, "completions/mean_length": 740.983349609375, "completions/mean_terminated_length": 740.983349609375, "completions/min_length": 660.2, "completions/min_terminated_length": 660.2, "entropy": 0.6699490308761596, "epoch": 0.038253825382538256, "frac_reward_zero_std": 0.0, "grad_norm": 0.71875, "learning_rate": 3.908872901678657e-07, "loss": -0.0133, "num_tokens": 28411772.0, "reward": 0.30716822743415834, "reward_std": 0.2654102027416229, "rewards/ADEnReward/mean": 0.028111883625388145, "rewards/ADEnReward/std": 0.07746810987591743, "rewards/CriticV2Reward/mean": 0.18488966822624206, "rewards/CriticV2Reward/std": 0.24855301678180694, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.2594019651412964, "sampling/importance_sampling_ratio/max": 2.6309361457824707, "sampling/importance_sampling_ratio/mean": 0.33823771476745607, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.0065004587173463, "sampling/sampling_logp_difference/mean": 0.02728327251970768, "step": 255, "step_time": 14.422917295061051 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 945.4, "completions/max_terminated_length": 945.4, "completions/mean_length": 726.4000244140625, "completions/mean_terminated_length": 726.4000244140625, "completions/min_length": 463.6, "completions/min_terminated_length": 463.6, "entropy": 0.6654939651489258, "epoch": 0.039003900390039, "frac_reward_zero_std": 0.0, "grad_norm": 0.5546875, "learning_rate": 3.7889688249400476e-07, "loss": -0.0193, "num_tokens": 28967004.0, "reward": 0.26902028620243074, "reward_std": 0.26705058515071867, "rewards/ADEnReward/mean": 0.030168159306049346, "rewards/ADEnReward/std": 0.07774421945214272, "rewards/CriticV2Reward/mean": 0.1463521108031273, "rewards/CriticV2Reward/std": 0.24327176213264465, "rewards/StrictFormatReward/mean": 0.925000011920929, "rewards/StrictFormatReward/std": 0.34017834067344666, "sampling/importance_sampling_ratio/max": 2.629823637008667, "sampling/importance_sampling_ratio/mean": 0.41424151659011843, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.1681773185729982, "sampling/sampling_logp_difference/mean": 0.026383167132735252, "step": 260, "step_time": 14.161155000329018 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 819.4, "completions/max_terminated_length": 819.4, "completions/mean_length": 730.5958618164062, "completions/mean_terminated_length": 730.5958618164062, "completions/min_length": 657.2, "completions/min_terminated_length": 657.2, "entropy": 0.6704589664936066, "epoch": 0.039753975397539755, "frac_reward_zero_std": 0.0, "grad_norm": 0.921875, "learning_rate": 3.6690647482014385e-07, "loss": -0.0642, "num_tokens": 29523163.0, "reward": 0.27605919539928436, "reward_std": 0.25954122841358185, "rewards/ADEnReward/mean": 0.031233209744095804, "rewards/ADEnReward/std": 0.08471837416291236, "rewards/CriticV2Reward/mean": 0.14815930277109146, "rewards/CriticV2Reward/std": 0.23515858948230745, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.5922812461853026, "sampling/importance_sampling_ratio/mean": 0.3501934826374054, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.589424896240234, "sampling/sampling_logp_difference/mean": 0.027005475759506226, "step": 265, "step_time": 13.391926497220993 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 817.6, "completions/max_terminated_length": 817.6, "completions/mean_length": 729.383349609375, "completions/mean_terminated_length": 729.383349609375, "completions/min_length": 662.0, "completions/min_terminated_length": 662.0, "entropy": 0.6677271962165833, "epoch": 0.0405040504050405, "frac_reward_zero_std": 0.0, "grad_norm": 0.3828125, "learning_rate": 3.54916067146283e-07, "loss": -0.0447, "num_tokens": 30079095.0, "reward": 0.2694405734539032, "reward_std": 0.27749488353729246, "rewards/ADEnReward/mean": 0.03271749503910541, "rewards/ADEnReward/std": 0.06845742091536522, "rewards/CriticV2Reward/mean": 0.141723071038723, "rewards/CriticV2Reward/std": 0.25534985661506654, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.23636061549186707, "sampling/importance_sampling_ratio/max": 2.580633020401001, "sampling/importance_sampling_ratio/mean": 0.35976060628890993, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.106691312789917, "sampling/sampling_logp_difference/mean": 0.027051576972007753, "step": 270, "step_time": 13.430866013094782 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1107.4, "completions/max_terminated_length": 815.8, "completions/mean_length": 745.3666870117188, "completions/mean_terminated_length": 738.6911376953125, "completions/min_length": 663.6, "completions/min_terminated_length": 663.6, "entropy": 0.7063815534114838, "epoch": 0.041254125412541254, "frac_reward_zero_std": 0.0, "grad_norm": 1.1875, "learning_rate": 3.4292565947242203e-07, "loss": -0.0802, "num_tokens": 30638991.0, "reward": 0.23000751733779906, "reward_std": 0.20237331390380858, "rewards/ADEnReward/mean": 0.014900790061801672, "rewards/ADEnReward/std": 0.047509999200701716, "rewards/CriticV2Reward/mean": 0.12094003856182098, "rewards/CriticV2Reward/std": 0.19368057399988176, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.294095641374588, "sampling/importance_sampling_ratio/max": 2.152031898498535, "sampling/importance_sampling_ratio/mean": 0.31464290618896484, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.139720010757446, "sampling/sampling_logp_difference/mean": 0.028014734387397766, "step": 275, "step_time": 15.272126619890333 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 966.2, "completions/max_terminated_length": 820.0, "completions/mean_length": 737.6000244140625, "completions/mean_terminated_length": 734.2439819335938, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "entropy": 0.6556420803070069, "epoch": 0.04200420042004201, "frac_reward_zero_std": 0.0, "grad_norm": 0.53125, "learning_rate": 3.309352517985611e-07, "loss": 0.0064, "num_tokens": 31197103.0, "reward": 0.28544660210609435, "reward_std": 0.2403573751449585, "rewards/ADEnReward/mean": 0.038210281357169154, "rewards/ADEnReward/std": 0.08102711364626884, "rewards/CriticV2Reward/mean": 0.1522363118827343, "rewards/CriticV2Reward/std": 0.21875708252191545, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.23636061549186707, "sampling/importance_sampling_ratio/max": 2.7030261993408202, "sampling/importance_sampling_ratio/mean": 0.41297419667243956, "sampling/importance_sampling_ratio/min": 2.0399140339577572e-06, "sampling/sampling_logp_difference/max": 3.274674487113953, "sampling/sampling_logp_difference/mean": 0.027079901099205016, "step": 280, "step_time": 14.378617286682129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 936.2, "completions/max_terminated_length": 936.2, "completions/mean_length": 737.020849609375, "completions/mean_terminated_length": 737.020849609375, "completions/min_length": 661.4, "completions/min_terminated_length": 661.4, "entropy": 0.6838120698928833, "epoch": 0.04275427542754275, "frac_reward_zero_std": 0.0, "grad_norm": 0.734375, "learning_rate": 3.1894484412470026e-07, "loss": -0.0648, "num_tokens": 31755108.0, "reward": 0.25459670424461367, "reward_std": 0.23896411657333375, "rewards/ADEnReward/mean": 0.022891263756901026, "rewards/ADEnReward/std": 0.04410543087869882, "rewards/CriticV2Reward/mean": 0.14253875762224197, "rewards/CriticV2Reward/std": 0.2287704974412918, "rewards/StrictFormatReward/mean": 0.8916666746139527, "rewards/StrictFormatReward/std": 0.3999585211277008, "sampling/importance_sampling_ratio/max": 2.5413411378860475, "sampling/importance_sampling_ratio/mean": 0.3536586403846741, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.051635646820069, "sampling/sampling_logp_difference/mean": 0.027624154463410378, "step": 285, "step_time": 14.16935004722327 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 810.4, "completions/max_terminated_length": 810.4, "completions/mean_length": 724.9750244140625, "completions/mean_terminated_length": 724.9750244140625, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "entropy": 0.6491694033145905, "epoch": 0.043504350435043505, "frac_reward_zero_std": 0.0, "grad_norm": 1.109375, "learning_rate": 3.069544364508393e-07, "loss": -0.0165, "num_tokens": 32309934.0, "reward": 0.26752397418022156, "reward_std": 0.22953160405158995, "rewards/ADEnReward/mean": 0.03401634339243174, "rewards/ADEnReward/std": 0.06949230469763279, "rewards/CriticV2Reward/mean": 0.13684094697237015, "rewards/CriticV2Reward/std": 0.21767082214355468, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.357547092437744, "sampling/importance_sampling_ratio/mean": 0.40171497464179995, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.093617391586304, "sampling/sampling_logp_difference/mean": 0.026295433193445204, "step": 290, "step_time": 13.348791869916022 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 797.8, "completions/max_terminated_length": 797.8, "completions/mean_length": 727.5416870117188, "completions/mean_terminated_length": 727.5416870117188, "completions/min_length": 673.6, "completions/min_terminated_length": 673.6, "entropy": 0.6584029793739319, "epoch": 0.04425442544254425, "frac_reward_zero_std": 0.0, "grad_norm": 0.9296875, "learning_rate": 2.949640287769784e-07, "loss": -0.0725, "num_tokens": 32865616.0, "reward": 0.3513562619686127, "reward_std": 0.32367265224456787, "rewards/ADEnReward/mean": 0.050219583325088024, "rewards/ADEnReward/std": 0.0958492573350668, "rewards/CriticV2Reward/mean": 0.20196999311447145, "rewards/CriticV2Reward/std": 0.2919680178165436, "rewards/StrictFormatReward/mean": 0.9916666626930237, "rewards/StrictFormatReward/std": 0.05773502588272095, "sampling/importance_sampling_ratio/max": 2.3279226541519167, "sampling/importance_sampling_ratio/mean": 0.39847335815429685, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.63114709854126, "sampling/sampling_logp_difference/mean": 0.026548907533288, "step": 295, "step_time": 13.368336435966194 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1089.0, "completions/max_terminated_length": 940.2, "completions/mean_length": 737.7500122070312, "completions/mean_terminated_length": 734.3935302734375, "completions/min_length": 662.2, "completions/min_terminated_length": 662.2, "entropy": 0.6550628364086151, "epoch": 0.045004500450045004, "frac_reward_zero_std": 0.0, "grad_norm": 0.61328125, "learning_rate": 2.8297362110311753e-07, "loss": 0.035, "num_tokens": 33423668.0, "reward": 0.27340781688690186, "reward_std": 0.25819830000400545, "rewards/ADEnReward/mean": 0.028038720414042474, "rewards/ADEnReward/std": 0.06851784735918046, "rewards/CriticV2Reward/mean": 0.15370242297649384, "rewards/CriticV2Reward/std": 0.23523760139942168, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.34808469414710996, "sampling/importance_sampling_ratio/max": 2.835608148574829, "sampling/importance_sampling_ratio/mean": 0.45533260703086853, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.564956688880921, "sampling/sampling_logp_difference/mean": 0.02656121402978897, "step": 300, "step_time": 15.144700674340129 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 798.0, "completions/max_terminated_length": 798.0, "completions/mean_length": 733.9666870117187, "completions/mean_terminated_length": 733.9666870117187, "completions/min_length": 669.2, "completions/min_terminated_length": 669.2, "entropy": 0.6536436915397644, "epoch": 0.04575457545754576, "frac_reward_zero_std": 0.0, "grad_norm": 0.8203125, "learning_rate": 2.7098321342925657e-07, "loss": -0.0131, "num_tokens": 33981308.0, "reward": 0.2970129132270813, "reward_std": 0.2672927647829056, "rewards/ADEnReward/mean": 0.016280055139213802, "rewards/ADEnReward/std": 0.04426044039428234, "rewards/CriticV2Reward/mean": 0.18656616508960724, "rewards/CriticV2Reward/std": 0.2617345839738846, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.2594019651412964, "sampling/importance_sampling_ratio/max": 2.434844732284546, "sampling/importance_sampling_ratio/mean": 0.3584721446037292, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.2410639762878417, "sampling/sampling_logp_difference/mean": 0.027303647249937057, "step": 305, "step_time": 13.214909846894443 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 946.2, "completions/max_terminated_length": 801.4, "completions/mean_length": 729.5666748046875, "completions/mean_terminated_length": 726.2310302734375, "completions/min_length": 666.0, "completions/min_terminated_length": 666.0, "entropy": 0.6449627339839935, "epoch": 0.0465046504650465, "frac_reward_zero_std": 0.0, "grad_norm": 0.462890625, "learning_rate": 2.5899280575539566e-07, "loss": 0.0087, "num_tokens": 34537204.0, "reward": 0.27850418388843534, "reward_std": 0.2546063870191574, "rewards/ADEnReward/mean": 0.03761259010061622, "rewards/ADEnReward/std": 0.07763511538505555, "rewards/CriticV2Reward/mean": 0.14589156806468964, "rewards/CriticV2Reward/std": 0.23430478870868682, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.27702280282974245, "sampling/importance_sampling_ratio/max": 2.2312597751617433, "sampling/importance_sampling_ratio/mean": 0.3535082578659058, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.5829816341400145, "sampling/sampling_logp_difference/mean": 0.026253899559378624, "step": 310, "step_time": 14.251038361247629 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1111.2, "completions/max_terminated_length": 961.2, "completions/mean_length": 731.0833618164063, "completions/mean_terminated_length": 727.6387573242188, "completions/min_length": 425.6, "completions/min_terminated_length": 425.6, "entropy": 0.6603303372859954, "epoch": 0.047254725472547256, "frac_reward_zero_std": 0.0, "grad_norm": 0.404296875, "learning_rate": 2.4700239808153475e-07, "loss": -0.0003, "num_tokens": 35093256.0, "reward": 0.3126928746700287, "reward_std": 0.2894691824913025, "rewards/ADEnReward/mean": 0.031799875758588315, "rewards/ADEnReward/std": 0.07806130312383175, "rewards/CriticV2Reward/mean": 0.18839299976825713, "rewards/CriticV2Reward/std": 0.2632020592689514, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.3342098295688629, "sampling/importance_sampling_ratio/max": 2.1868972778320312, "sampling/importance_sampling_ratio/mean": 0.3813866376876831, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.841550922393799, "sampling/sampling_logp_difference/mean": 0.027124195545911788, "step": 315, "step_time": 15.209461131505668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 834.2, "completions/max_terminated_length": 834.2, "completions/mean_length": 731.2291870117188, "completions/mean_terminated_length": 731.2291870117188, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "entropy": 0.6608487665653229, "epoch": 0.048004800480048, "frac_reward_zero_std": 0.0, "grad_norm": 0.67578125, "learning_rate": 2.3501199040767386e-07, "loss": -0.0772, "num_tokens": 35649903.0, "reward": 0.3282505989074707, "reward_std": 0.2977394014596939, "rewards/ADEnReward/mean": 0.03128300998359919, "rewards/ADEnReward/std": 0.061178040504455564, "rewards/CriticV2Reward/mean": 0.2028009206056595, "rewards/CriticV2Reward/std": 0.2806028872728348, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.3347578287124634, "sampling/importance_sampling_ratio/max": 2.501104164123535, "sampling/importance_sampling_ratio/mean": 0.42977511286735537, "sampling/importance_sampling_ratio/min": 0.0003163336776196957, "sampling/sampling_logp_difference/max": 3.5600287437438967, "sampling/sampling_logp_difference/mean": 0.02700127139687538, "step": 320, "step_time": 13.359743607882411 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 818.2, "completions/max_terminated_length": 818.2, "completions/mean_length": 733.5458374023438, "completions/mean_terminated_length": 733.5458374023438, "completions/min_length": 652.6, "completions/min_terminated_length": 652.6, "entropy": 0.688176554441452, "epoch": 0.048754875487548754, "frac_reward_zero_std": 0.0, "grad_norm": 0.302734375, "learning_rate": 2.2302158273381293e-07, "loss": 0.053, "num_tokens": 36206706.0, "reward": 0.24143614768981933, "reward_std": 0.2442352443933487, "rewards/ADEnReward/mean": 0.02315669395029545, "rewards/ADEnReward/std": 0.06468662098050118, "rewards/CriticV2Reward/mean": 0.1241127759218216, "rewards/CriticV2Reward/std": 0.21812452077865602, "rewards/StrictFormatReward/mean": 0.9416666626930237, "rewards/StrictFormatReward/std": 0.25023548007011415, "sampling/importance_sampling_ratio/max": 2.460072135925293, "sampling/importance_sampling_ratio/mean": 0.4378948867321014, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.8208963990211486, "sampling/sampling_logp_difference/mean": 0.0267129085958004, "step": 325, "step_time": 13.40539922695607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 849.6, "completions/max_terminated_length": 849.6, "completions/mean_length": 736.9750366210938, "completions/mean_terminated_length": 736.9750366210938, "completions/min_length": 552.2, "completions/min_terminated_length": 552.2, "entropy": 0.6794994473457336, "epoch": 0.04950495049504951, "frac_reward_zero_std": 0.0, "grad_norm": 1.046875, "learning_rate": 2.1103117505995202e-07, "loss": -0.022, "num_tokens": 36764908.0, "reward": 0.33096388876438143, "reward_std": 0.29873789846897125, "rewards/ADEnReward/mean": 0.029305444657802583, "rewards/ADEnReward/std": 0.07149080038070679, "rewards/CriticV2Reward/mean": 0.20999177694320678, "rewards/CriticV2Reward/std": 0.275063157081604, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.3074225068092346, "sampling/importance_sampling_ratio/max": 2.370104718208313, "sampling/importance_sampling_ratio/mean": 0.3515215128660202, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 5.771962738037109, "sampling/sampling_logp_difference/mean": 0.028216013684868813, "step": 330, "step_time": 13.547085477039218 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 909.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 736.2750122070313, "completions/mean_terminated_length": 736.2750122070313, "completions/min_length": 595.6, "completions/min_terminated_length": 595.6, "entropy": 0.6611269891262055, "epoch": 0.05025502550255025, "frac_reward_zero_std": 0.0, "grad_norm": 0.94921875, "learning_rate": 1.9904076738609113e-07, "loss": -0.0151, "num_tokens": 37322686.0, "reward": 0.3241033911705017, "reward_std": 0.2873489558696747, "rewards/ADEnReward/mean": 0.031790857389569285, "rewards/ADEnReward/std": 0.07606348469853401, "rewards/CriticV2Reward/mean": 0.19981251955032348, "rewards/CriticV2Reward/std": 0.2744706034660339, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.37487201690673827, "sampling/importance_sampling_ratio/max": 2.115146613121033, "sampling/importance_sampling_ratio/mean": 0.34482905715703965, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.7548720836639404, "sampling/sampling_logp_difference/mean": 0.027143753319978713, "step": 335, "step_time": 13.92563083646819 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1094.2, "completions/max_terminated_length": 943.2, "completions/mean_length": 740.1625244140625, "completions/mean_terminated_length": 736.7945190429688, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.6796614170074463, "epoch": 0.051005100510051006, "frac_reward_zero_std": 0.0, "grad_norm": 0.3203125, "learning_rate": 1.870503597122302e-07, "loss": -0.0118, "num_tokens": 37881477.0, "reward": 0.2684012562036514, "reward_std": 0.2399124890565872, "rewards/ADEnReward/mean": 0.022319978382438422, "rewards/ADEnReward/std": 0.05877099372446537, "rewards/CriticV2Reward/mean": 0.15108125582337378, "rewards/CriticV2Reward/std": 0.22448689192533494, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.23636061549186707, "sampling/importance_sampling_ratio/max": 2.3924092769622805, "sampling/importance_sampling_ratio/mean": 0.323573100566864, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 4.640535640716553, "sampling/sampling_logp_difference/mean": 0.027076425030827522, "step": 340, "step_time": 15.11005504000932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 902.8, "completions/max_terminated_length": 902.8, "completions/mean_length": 733.1625244140625, "completions/mean_terminated_length": 733.1625244140625, "completions/min_length": 675.4, "completions/min_terminated_length": 675.4, "entropy": 0.6722322583198548, "epoch": 0.05175517551755175, "frac_reward_zero_std": 0.0, "grad_norm": 0.62890625, "learning_rate": 1.750599520383693e-07, "loss": 0.0122, "num_tokens": 38438508.0, "reward": 0.2832422822713852, "reward_std": 0.2662737250328064, "rewards/ADEnReward/mean": 0.01820658575743437, "rewards/ADEnReward/std": 0.051234375685453415, "rewards/CriticV2Reward/mean": 0.16503570079803467, "rewards/CriticV2Reward/std": 0.25538637936115266, "rewards/StrictFormatReward/mean": 1.0, "rewards/StrictFormatReward/std": 0.0, "sampling/importance_sampling_ratio/max": 2.2767002105712892, "sampling/importance_sampling_ratio/mean": 0.4129745185375214, "sampling/importance_sampling_ratio/min": 0.0010848915204405784, "sampling/sampling_logp_difference/max": 1.5775320053100585, "sampling/sampling_logp_difference/mean": 0.0266124464571476, "step": 345, "step_time": 13.816253677289932 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 795.6, "completions/max_terminated_length": 795.6, "completions/mean_length": 726.17919921875, "completions/mean_terminated_length": 726.17919921875, "completions/min_length": 548.2, "completions/min_terminated_length": 548.2, "entropy": 0.6423774480819702, "epoch": 0.052505250525052505, "frac_reward_zero_std": 0.0, "grad_norm": 0.427734375, "learning_rate": 1.630695443645084e-07, "loss": -0.0081, "num_tokens": 38993991.0, "reward": 0.3389123469591141, "reward_std": 0.29732525944709776, "rewards/ADEnReward/mean": 0.04917324259877205, "rewards/ADEnReward/std": 0.09704692177474498, "rewards/CriticV2Reward/mean": 0.19807242900133132, "rewards/CriticV2Reward/std": 0.2688820421695709, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.39194485545158386, "sampling/importance_sampling_ratio/max": 2.469262218475342, "sampling/importance_sampling_ratio/mean": 0.41353853344917296, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.8893362045288087, "sampling/sampling_logp_difference/mean": 0.026003163307905197, "step": 350, "step_time": 13.209398588072508 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 952.4, "completions/max_terminated_length": 811.4, "completions/mean_length": 732.6875244140625, "completions/mean_terminated_length": 729.3064819335938, "completions/min_length": 670.4, "completions/min_terminated_length": 670.4, "entropy": 0.6782492220401763, "epoch": 0.05325532553255326, "frac_reward_zero_std": 0.0, "grad_norm": 0.265625, "learning_rate": 1.5107913669064747e-07, "loss": -0.0119, "num_tokens": 39550364.0, "reward": 0.24834446907043456, "reward_std": 0.22241731882095336, "rewards/ADEnReward/mean": 0.018331396393477917, "rewards/ADEnReward/std": 0.049924323707818984, "rewards/CriticV2Reward/mean": 0.13501306772232055, "rewards/CriticV2Reward/std": 0.21232716143131256, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.23636061549186707, "sampling/importance_sampling_ratio/max": 2.49073007106781, "sampling/importance_sampling_ratio/mean": 0.37095364928245544, "sampling/importance_sampling_ratio/min": 0.0006488471291959286, "sampling/sampling_logp_difference/max": 2.1764797449111937, "sampling/sampling_logp_difference/mean": 0.02676672637462616, "step": 355, "step_time": 14.224239609390498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 1073.6, "completions/max_terminated_length": 925.2, "completions/mean_length": 734.620849609375, "completions/mean_terminated_length": 731.280419921875, "completions/min_length": 437.4, "completions/min_terminated_length": 437.4, "entropy": 0.6800269901752471, "epoch": 0.054005400540054004, "frac_reward_zero_std": 0.0, "grad_norm": 0.455078125, "learning_rate": 1.3908872901678656e-07, "loss": 0.0046, "num_tokens": 40107809.0, "reward": 0.24553753435611725, "reward_std": 0.21728258728981018, "rewards/ADEnReward/mean": 0.02120685214176774, "rewards/ADEnReward/std": 0.05041029825806618, "rewards/CriticV2Reward/mean": 0.13183067739009857, "rewards/CriticV2Reward/std": 0.2057559758424759, "rewards/StrictFormatReward/mean": 0.925000011920929, "rewards/StrictFormatReward/std": 0.34017834067344666, "sampling/importance_sampling_ratio/max": 2.6524218559265136, "sampling/importance_sampling_ratio/mean": 0.4359212100505829, "sampling/importance_sampling_ratio/min": 0.00012990221148356795, "sampling/sampling_logp_difference/max": 4.0367347240448, "sampling/sampling_logp_difference/mean": 0.02658579796552658, "step": 360, "step_time": 15.101525206118822 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 807.2, "completions/max_terminated_length": 807.2, "completions/mean_length": 728.8291870117188, "completions/mean_terminated_length": 728.8291870117188, "completions/min_length": 439.6, "completions/min_terminated_length": 439.6, "entropy": 0.6668968260288238, "epoch": 0.054755475547554756, "frac_reward_zero_std": 0.0, "grad_norm": 1.1171875, "learning_rate": 1.2709832134292565e-07, "loss": -0.0227, "num_tokens": 40663784.0, "reward": 0.29526926279067994, "reward_std": 0.2813252925872803, "rewards/ADEnReward/mean": 0.03403082452714443, "rewards/ADEnReward/std": 0.08168720752000809, "rewards/CriticV2Reward/mean": 0.16957174986600876, "rewards/CriticV2Reward/std": 0.24872368276119233, "rewards/StrictFormatReward/mean": 0.9166666626930237, "rewards/StrictFormatReward/std": 0.34808469414710996, "sampling/importance_sampling_ratio/max": 2.390850281715393, "sampling/importance_sampling_ratio/mean": 0.4092385768890381, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.3626219749450685, "sampling/sampling_logp_difference/mean": 0.026888956502079963, "step": 365, "step_time": 13.315052114892751 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 804.2, "completions/max_terminated_length": 804.2, "completions/mean_length": 733.4416870117187, "completions/mean_terminated_length": 733.4416870117187, "completions/min_length": 663.8, "completions/min_terminated_length": 663.8, "entropy": 0.6606446027755737, "epoch": 0.0555055505550555, "frac_reward_zero_std": 0.0, "grad_norm": 0.78125, "learning_rate": 1.1510791366906475e-07, "loss": -0.0329, "num_tokens": 41220978.0, "reward": 0.2930675059556961, "reward_std": 0.2800747185945511, "rewards/ADEnReward/mean": 0.03040240630507469, "rewards/ADEnReward/std": 0.07180959656834603, "rewards/CriticV2Reward/mean": 0.16849841624498368, "rewards/CriticV2Reward/std": 0.25502766370773317, "rewards/StrictFormatReward/mean": 0.9416666746139526, "rewards/StrictFormatReward/std": 0.30006415247917173, "sampling/importance_sampling_ratio/max": 2.7124672889709474, "sampling/importance_sampling_ratio/mean": 0.43735997676849364, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.8521456480026246, "sampling/sampling_logp_difference/mean": 0.02668215483427048, "step": 370, "step_time": 13.217574856709689 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 803.4, "completions/max_terminated_length": 803.4, "completions/mean_length": 732.1750244140625, "completions/mean_terminated_length": 732.1750244140625, "completions/min_length": 664.8, "completions/min_terminated_length": 664.8, "entropy": 0.6729780435562134, "epoch": 0.056255625562556255, "frac_reward_zero_std": 0.0, "grad_norm": 0.6875, "learning_rate": 1.0311750599520383e-07, "loss": 0.057, "num_tokens": 41777820.0, "reward": 0.292266783118248, "reward_std": 0.2692907273769379, "rewards/ADEnReward/mean": 0.02988054435700178, "rewards/ADEnReward/std": 0.0724082849919796, "rewards/CriticV2Reward/mean": 0.16571956276893615, "rewards/CriticV2Reward/std": 0.25571154057979584, "rewards/StrictFormatReward/mean": 0.9666666626930237, "rewards/StrictFormatReward/std": 0.19624642729759217, "sampling/importance_sampling_ratio/max": 2.654768705368042, "sampling/importance_sampling_ratio/mean": 0.4235609233379364, "sampling/importance_sampling_ratio/min": 0.000751539459452033, "sampling/sampling_logp_difference/max": 3.4154165744781495, "sampling/sampling_logp_difference/mean": 0.027747052907943725, "step": 375, "step_time": 13.223657737579197 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 946.8, "completions/max_terminated_length": 946.8, "completions/mean_length": 729.4666870117187, "completions/mean_terminated_length": 729.4666870117187, "completions/min_length": 556.4, "completions/min_terminated_length": 556.4, "entropy": 0.6680528521537781, "epoch": 0.05700570057005701, "frac_reward_zero_std": 0.0, "grad_norm": 0.90234375, "learning_rate": 9.112709832134293e-08, "loss": 0.0034, "num_tokens": 42333820.0, "reward": 0.30099524557590485, "reward_std": 0.28950676918029783, "rewards/ADEnReward/mean": 0.04762665368616581, "rewards/ADEnReward/std": 0.11501347124576569, "rewards/CriticV2Reward/mean": 0.16253525763750076, "rewards/CriticV2Reward/std": 0.23468865156173707, "rewards/StrictFormatReward/mean": 0.9083333373069763, "rewards/StrictFormatReward/std": 0.4005065202713013, "sampling/importance_sampling_ratio/max": 2.560587167739868, "sampling/importance_sampling_ratio/mean": 0.37538244128227233, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4054880142211914, "sampling/sampling_logp_difference/mean": 0.02632361464202404, "step": 380, "step_time": 14.30399916516617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 832.2, "completions/max_terminated_length": 832.2, "completions/mean_length": 735.6333618164062, "completions/mean_terminated_length": 735.6333618164062, "completions/min_length": 652.8, "completions/min_terminated_length": 652.8, "entropy": 0.6726574301719666, "epoch": 0.057755775577557754, "frac_reward_zero_std": 0.0, "grad_norm": 0.609375, "learning_rate": 7.913669064748201e-08, "loss": -0.048, "num_tokens": 42891412.0, "reward": 0.30219454765319825, "reward_std": 0.27768056392669677, "rewards/ADEnReward/mean": 0.03429584912955761, "rewards/ADEnReward/std": 0.07258765399456024, "rewards/CriticV2Reward/mean": 0.17539868950843812, "rewards/CriticV2Reward/std": 0.2650424838066101, "rewards/StrictFormatReward/mean": 0.925000011920929, "rewards/StrictFormatReward/std": 0.34017834067344666, "sampling/importance_sampling_ratio/max": 2.453314113616943, "sampling/importance_sampling_ratio/mean": 0.3553328990936279, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.31168155670166, "sampling/sampling_logp_difference/mean": 0.02763594649732113, "step": 385, "step_time": 13.449316807556897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 947.2, "completions/max_terminated_length": 947.2, "completions/mean_length": 739.39169921875, "completions/mean_terminated_length": 739.39169921875, "completions/min_length": 671.8, "completions/min_terminated_length": 671.8, "entropy": 0.6789502263069153, "epoch": 0.05850585058505851, "frac_reward_zero_std": 0.0, "grad_norm": 0.474609375, "learning_rate": 6.71462829736211e-08, "loss": -0.0058, "num_tokens": 43450082.0, "reward": 0.2598124861717224, "reward_std": 0.2435460090637207, "rewards/ADEnReward/mean": 0.04101892188191414, "rewards/ADEnReward/std": 0.09139346182346345, "rewards/CriticV2Reward/mean": 0.12296021729707718, "rewards/CriticV2Reward/std": 0.21000217646360397, "rewards/StrictFormatReward/mean": 0.9583333373069763, "rewards/StrictFormatReward/std": 0.21928777694702148, "sampling/importance_sampling_ratio/max": 2.6636743545532227, "sampling/importance_sampling_ratio/mean": 0.41307313442230226, "sampling/importance_sampling_ratio/min": 0.00033994775731116533, "sampling/sampling_logp_difference/max": 4.322899675369262, "sampling/sampling_logp_difference/mean": 0.026649556308984756, "step": 390, "step_time": 14.210391772259026 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 821.2, "completions/max_terminated_length": 821.2, "completions/mean_length": 729.7666870117188, "completions/mean_terminated_length": 729.7666870117188, "completions/min_length": 554.6, "completions/min_terminated_length": 554.6, "entropy": 0.6675756096839904, "epoch": 0.05925592559255925, "frac_reward_zero_std": 0.0, "grad_norm": 0.671875, "learning_rate": 5.5155875299760186e-08, "loss": 0.0211, "num_tokens": 44006314.0, "reward": 0.28109568953514097, "reward_std": 0.25426478683948517, "rewards/ADEnReward/mean": 0.01611993897240609, "rewards/ADEnReward/std": 0.039170157397165895, "rewards/CriticV2Reward/mean": 0.16997574269771576, "rewards/CriticV2Reward/std": 0.24328334629535675, "rewards/StrictFormatReward/mean": 0.949999988079071, "rewards/StrictFormatReward/std": 0.31171647906303407, "sampling/importance_sampling_ratio/max": 2.653896713256836, "sampling/importance_sampling_ratio/mean": 0.4388413727283478, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 1.7347460508346557, "sampling/sampling_logp_difference/mean": 0.027177633345127107, "step": 395, "step_time": 13.348359900712968 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00833333358168602, "completions/max_length": 1113.0, "completions/max_terminated_length": 824.8, "completions/mean_length": 737.6625244140625, "completions/mean_terminated_length": 730.9420166015625, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "entropy": 0.6727102100849152, "epoch": 0.060006000600060005, "frac_reward_zero_std": 0.0, "grad_norm": 0.451171875, "learning_rate": 4.316546762589928e-08, "loss": -0.0499, "num_tokens": 44564569.0, "reward": 0.31804165840148924, "reward_std": 0.26383175849914553, "rewards/ADEnReward/mean": 0.0430668905377388, "rewards/ADEnReward/std": 0.08763031214475632, "rewards/CriticV2Reward/mean": 0.18247474879026412, "rewards/CriticV2Reward/std": 0.24957966804504395, "rewards/StrictFormatReward/mean": 0.925, "rewards/StrictFormatReward/std": 0.37487201690673827, "sampling/importance_sampling_ratio/max": 2.1602240800857544, "sampling/importance_sampling_ratio/mean": 0.3833409368991852, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.4215245723724363, "sampling/sampling_logp_difference/mean": 0.02680293843150139, "step": 400, "step_time": 15.337327389977872 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 806.8, "completions/max_terminated_length": 806.8, "completions/mean_length": 733.6291748046875, "completions/mean_terminated_length": 733.6291748046875, "completions/min_length": 655.4, "completions/min_terminated_length": 655.4, "entropy": 0.6855071008205413, "epoch": 0.06075607560756076, "frac_reward_zero_std": 0.0, "grad_norm": 0.89453125, "learning_rate": 3.1175059952038366e-08, "loss": 0.0418, "num_tokens": 45121504.0, "reward": 0.30168403387069703, "reward_std": 0.26787208914756777, "rewards/ADEnReward/mean": 0.029562235996127128, "rewards/ADEnReward/std": 0.0694606777280569, "rewards/CriticV2Reward/mean": 0.17628844380378722, "rewards/CriticV2Reward/std": 0.2518341809511185, "rewards/StrictFormatReward/mean": 0.9583333373069763, "rewards/StrictFormatReward/std": 0.21928777694702148, "sampling/importance_sampling_ratio/max": 2.63645486831665, "sampling/importance_sampling_ratio/mean": 0.37098753452301025, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 2.670264649391174, "sampling/sampling_logp_difference/mean": 0.027659954503178596, "step": 405, "step_time": 13.189621604420244 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.00416666679084301, "completions/max_length": 956.2, "completions/max_terminated_length": 810.6, "completions/mean_length": 735.0416870117188, "completions/mean_terminated_length": 731.6653564453125, "completions/min_length": 650.8, "completions/min_terminated_length": 650.8, "entropy": 0.6711634755134582, "epoch": 0.061506150615061504, "frac_reward_zero_std": 0.0, "grad_norm": 0.73828125, "learning_rate": 1.9184652278177456e-08, "loss": -0.0293, "num_tokens": 45678922.0, "reward": 0.27163800299167634, "reward_std": 0.26216488778591157, "rewards/ADEnReward/mean": 0.014222659170627594, "rewards/ADEnReward/std": 0.03859034609049559, "rewards/CriticV2Reward/mean": 0.1624153345823288, "rewards/CriticV2Reward/std": 0.2478887140750885, "rewards/StrictFormatReward/mean": 0.95, "rewards/StrictFormatReward/std": 0.27702280282974245, "sampling/importance_sampling_ratio/max": 2.693353033065796, "sampling/importance_sampling_ratio/mean": 0.4223414957523346, "sampling/importance_sampling_ratio/min": 1.2648591791730722e-07, "sampling/sampling_logp_difference/max": 5.323451900482178, "sampling/sampling_logp_difference/mean": 0.026813267916440963, "step": 410, "step_time": 14.17747332798317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1070.4, "completions/max_terminated_length": 1070.4, "completions/mean_length": 739.8833618164062, "completions/mean_terminated_length": 739.8833618164062, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "entropy": 0.6690365135669708, "epoch": 0.06225622562256226, "frac_reward_zero_std": 0.0, "grad_norm": 0.703125, "learning_rate": 7.194244604316547e-09, "loss": -0.0379, "num_tokens": 46237678.0, "reward": 0.3154370367527008, "reward_std": 0.2778420358896255, "rewards/ADEnReward/mean": 0.033237920398823916, "rewards/ADEnReward/std": 0.0803776565939188, "rewards/CriticV2Reward/mean": 0.1888657659292221, "rewards/CriticV2Reward/std": 0.25663301944732664, "rewards/StrictFormatReward/mean": 0.9333333492279052, "rewards/StrictFormatReward/std": 0.32310550212860106, "sampling/importance_sampling_ratio/max": 2.434389925003052, "sampling/importance_sampling_ratio/mean": 0.3478035569190979, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 3.7522983074188234, "sampling/sampling_logp_difference/mean": 0.027356555312871934, "step": 415, "step_time": 15.047657939139754 } ], "logging_steps": 5, "max_steps": 417, "num_input_tokens_seen": 46460815, "num_train_epochs": 1, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 6, "trial_name": null, "trial_params": null }