{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 1.3888888888888888, "eval_steps": 500, "global_step": 500, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3453125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2711.7, "completions/mean_length": 3878.6890625, "completions/mean_terminated_length": 1619.041455078125, "completions/min_length": 609.9, "completions/min_terminated_length": 609.9, "entropy": 0.5342199512757361, "epoch": 0.027777777777777776, "frac_reward_zero_std": 0.275, "grad_norm": 0.2434006631374359, "learning_rate": 3e-07, "loss": -0.0252, "num_tokens": 2530705.0, "reward": 0.3828125, "reward_std": 0.4778271645307541, "rewards/reward_correct/mean": 0.3828125, "rewards/reward_correct/std": 0.47782717645168304, "sampling/importance_sampling_ratio/max": 2.229939651489258, "sampling/importance_sampling_ratio/mean": 0.28947882503271105, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 20.731588554382324, "sampling/sampling_logp_difference/mean": 0.042482714727520944, "step": 10, "step_time": 97.8341444697231 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2827.0, "completions/mean_length": 3604.1640625, "completions/mean_terminated_length": 1672.880029296875, "completions/min_length": 578.1, "completions/min_terminated_length": 578.1, "entropy": 0.5231569674331695, "epoch": 0.05555555555555555, "frac_reward_zero_std": 0.175, "grad_norm": 0.1266385316848755, "learning_rate": 6.333333333333332e-07, "loss": -0.0326, "num_tokens": 4884674.0, "reward": 0.4109375, "reward_std": 0.47415409684181214, "rewards/reward_correct/mean": 0.4109375, "rewards/reward_correct/std": 0.4741541087627411, "sampling/importance_sampling_ratio/max": 2.4683921217918394, "sampling/importance_sampling_ratio/mean": 0.3206924065947533, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.872397041320802, "sampling/sampling_logp_difference/mean": 0.04122307561337948, "step": 20, "step_time": 91.14647748004646 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.284375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2853.0, "completions/mean_length": 3462.8375, "completions/mean_terminated_length": 1606.2551391601562, "completions/min_length": 553.3, "completions/min_terminated_length": 553.3, "entropy": 0.489674887387082, "epoch": 0.08333333333333333, "frac_reward_zero_std": 0.25, "grad_norm": 0.10141309350728989, "learning_rate": 9.666666666666666e-07, "loss": -0.0178, "num_tokens": 7152210.0, "reward": 0.45, "reward_std": 0.4725195735692978, "rewards/reward_correct/mean": 0.45, "rewards/reward_correct/std": 0.47251958549022677, "sampling/importance_sampling_ratio/max": 2.1169467091560366, "sampling/importance_sampling_ratio/mean": 0.3329242721199989, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.969456291198732, "sampling/sampling_logp_difference/mean": 0.03882257957011461, "step": 30, "step_time": 89.79702748470008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2707.2, "completions/mean_length": 4290.6703125, "completions/mean_terminated_length": 1551.9008666992188, "completions/min_length": 537.6, "completions/min_terminated_length": 537.6, "entropy": 0.5838452530559153, "epoch": 0.1111111111111111, "frac_reward_zero_std": 0.25, "grad_norm": 0.11795759201049805, "learning_rate": 9.997876019358083e-07, "loss": -0.0185, "num_tokens": 9945431.0, "reward": 0.3640625, "reward_std": 0.4712675392627716, "rewards/reward_correct/mean": 0.3640625, "rewards/reward_correct/std": 0.471267557144165, "sampling/importance_sampling_ratio/max": 2.536797857284546, "sampling/importance_sampling_ratio/mean": 0.29656108319759367, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.390139389038087, "sampling/sampling_logp_difference/mean": 0.04604039713740349, "step": 40, "step_time": 104.82241893839091 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3453125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2767.5, "completions/mean_length": 3832.1796875, "completions/mean_terminated_length": 1559.965576171875, "completions/min_length": 624.6, "completions/min_terminated_length": 624.6, "entropy": 0.5342930512968451, "epoch": 0.1388888888888889, "frac_reward_zero_std": 0.3125, "grad_norm": 1.948204517364502, "learning_rate": 9.990536180750723e-07, "loss": -0.0307, "num_tokens": 12443042.0, "reward": 0.403125, "reward_std": 0.4718716889619827, "rewards/reward_correct/mean": 0.403125, "rewards/reward_correct/std": 0.4718716889619827, "sampling/importance_sampling_ratio/max": 2.631464684009552, "sampling/importance_sampling_ratio/mean": 0.3282940939068794, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.956416893005372, "sampling/sampling_logp_difference/mean": 0.045470394939184186, "step": 50, "step_time": 96.40531954467296 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.353125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2789.8, "completions/mean_length": 3885.1921875, "completions/mean_terminated_length": 1538.73271484375, "completions/min_length": 554.3, "completions/min_terminated_length": 554.3, "entropy": 0.5527781342156232, "epoch": 0.16666666666666666, "frac_reward_zero_std": 0.2125, "grad_norm": 0.2698710858821869, "learning_rate": 9.97796195883804e-07, "loss": -0.0329, "num_tokens": 14976877.0, "reward": 0.4359375, "reward_std": 0.4925714761018753, "rewards/reward_correct/mean": 0.4359375, "rewards/reward_correct/std": 0.49257148802280426, "sampling/importance_sampling_ratio/max": 2.5233256936073305, "sampling/importance_sampling_ratio/mean": 0.32925176024436953, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.101856422424316, "sampling/sampling_logp_difference/mean": 0.04616186544299126, "step": 60, "step_time": 97.68703336380422 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.346875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2801.9, "completions/mean_length": 3842.059375, "completions/mean_terminated_length": 1533.8720336914062, "completions/min_length": 606.3, "completions/min_terminated_length": 606.3, "entropy": 0.5389381644781679, "epoch": 0.19444444444444445, "frac_reward_zero_std": 0.275, "grad_norm": 0.1625521332025528, "learning_rate": 9.960166542242428e-07, "loss": -0.0281, "num_tokens": 17480451.0, "reward": 0.4265625, "reward_std": 0.4911981523036957, "rewards/reward_correct/mean": 0.4265625, "rewards/reward_correct/std": 0.49119815826416013, "sampling/importance_sampling_ratio/max": 2.4373825907707216, "sampling/importance_sampling_ratio/mean": 0.32550778836011884, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.449700927734376, "sampling/sampling_logp_difference/mean": 0.04671826101839542, "step": 70, "step_time": 96.85919274576008 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2702.0, "completions/mean_length": 3792.390625, "completions/mean_terminated_length": 1554.6982788085938, "completions/min_length": 523.0, "completions/min_terminated_length": 523.0, "entropy": 0.5755266681313514, "epoch": 0.2222222222222222, "frac_reward_zero_std": 0.275, "grad_norm": 0.2741508483886719, "learning_rate": 9.937168595898508e-07, "loss": -0.0216, "num_tokens": 19956309.0, "reward": 0.396875, "reward_std": 0.48693000376224516, "rewards/reward_correct/mean": 0.396875, "rewards/reward_correct/std": 0.48693000674247744, "sampling/importance_sampling_ratio/max": 2.2106072545051574, "sampling/importance_sampling_ratio/mean": 0.30533475875854493, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.59271354675293, "sampling/sampling_logp_difference/mean": 0.04881819523870945, "step": 80, "step_time": 95.62034683078528 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3796875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2839.8, "completions/mean_length": 4084.5625, "completions/mean_terminated_length": 1582.0239868164062, "completions/min_length": 598.2, "completions/min_terminated_length": 598.2, "entropy": 0.5755758906248957, "epoch": 0.25, "frac_reward_zero_std": 0.2375, "grad_norm": 0.11705684661865234, "learning_rate": 9.908992241476186e-07, "loss": -0.0328, "num_tokens": 22618117.0, "reward": 0.3671875, "reward_std": 0.46752808690071107, "rewards/reward_correct/mean": 0.3671875, "rewards/reward_correct/std": 0.4675280928611755, "sampling/importance_sampling_ratio/max": 2.1488356232643127, "sampling/importance_sampling_ratio/mean": 0.282461379468441, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.50088768005371, "sampling/sampling_logp_difference/mean": 0.04865880124270916, "step": 90, "step_time": 101.01973776686937 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4203125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2645.3, "completions/mean_length": 4314.771875, "completions/mean_terminated_length": 1509.4629516601562, "completions/min_length": 569.4, "completions/min_terminated_length": 569.4, "entropy": 0.5954360837582499, "epoch": 0.2777777777777778, "frac_reward_zero_std": 0.275, "grad_norm": 0.09120581299066544, "learning_rate": 9.875667032080352e-07, "loss": -0.0301, "num_tokens": 25430539.0, "reward": 0.353125, "reward_std": 0.4764838844537735, "rewards/reward_correct/mean": 0.353125, "rewards/reward_correct/std": 0.4764839023351669, "sampling/importance_sampling_ratio/max": 2.3067823886871337, "sampling/importance_sampling_ratio/mean": 0.28874450623989106, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.806869316101075, "sampling/sampling_logp_difference/mean": 0.050743330270051956, "step": 100, "step_time": 106.27121401969343 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2765625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2885.7, "completions/mean_length": 3313.5390625, "completions/mean_terminated_length": 1451.3173950195312, "completions/min_length": 524.1, "completions/min_terminated_length": 524.1, "entropy": 0.5162243159022182, "epoch": 0.3055555555555556, "frac_reward_zero_std": 0.25, "grad_norm": 0.20105834305286407, "learning_rate": 9.837227921253745e-07, "loss": -0.0222, "num_tokens": 27594124.0, "reward": 0.4359375, "reward_std": 0.48942558765411376, "rewards/reward_correct/mean": 0.4359375, "rewards/reward_correct/std": 0.4894255995750427, "sampling/importance_sampling_ratio/max": 2.4590456604957582, "sampling/importance_sampling_ratio/mean": 0.36281110197305677, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.5187047958374, "sampling/sampling_logp_difference/mean": 0.04329596869647503, "step": 110, "step_time": 88.54633442051708 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3703125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2659.6, "completions/mean_length": 3929.0671875, "completions/mean_terminated_length": 1439.959716796875, "completions/min_length": 555.3, "completions/min_terminated_length": 555.3, "entropy": 0.5905495826154947, "epoch": 0.3333333333333333, "frac_reward_zero_std": 0.2375, "grad_norm": 0.08803282678127289, "learning_rate": 9.793715226315528e-07, "loss": -0.0261, "num_tokens": 30159399.0, "reward": 0.421875, "reward_std": 0.47909272611141207, "rewards/reward_correct/mean": 0.421875, "rewards/reward_correct/std": 0.47909273505210875, "sampling/importance_sampling_ratio/max": 2.126016139984131, "sampling/importance_sampling_ratio/mean": 0.27617028504610064, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.618224716186525, "sampling/sampling_logp_difference/mean": 0.05163493566215038, "step": 120, "step_time": 99.54315063767135 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.378125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2735.7, "completions/mean_length": 4040.6890625, "completions/mean_terminated_length": 1536.42490234375, "completions/min_length": 645.0, "completions/min_terminated_length": 645.0, "entropy": 0.5655143281910568, "epoch": 0.3611111111111111, "frac_reward_zero_std": 0.2125, "grad_norm": 0.1966542899608612, "learning_rate": 9.745174586073982e-07, "loss": -0.0311, "num_tokens": 32795344.0, "reward": 0.384375, "reward_std": 0.47214375138282777, "rewards/reward_correct/mean": 0.384375, "rewards/reward_correct/std": 0.4721437692642212, "sampling/importance_sampling_ratio/max": 2.5283960938453673, "sampling/importance_sampling_ratio/mean": 0.3151987254619598, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.23869686126709, "sampling/sampling_logp_difference/mean": 0.049242925643920896, "step": 130, "step_time": 100.30694740768521 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2765625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2714.3, "completions/mean_length": 3301.6859375, "completions/mean_terminated_length": 1435.6876953125, "completions/min_length": 568.2, "completions/min_terminated_length": 568.2, "entropy": 0.5224350173491985, "epoch": 0.3888888888888889, "frac_reward_zero_std": 0.225, "grad_norm": 0.266464501619339, "learning_rate": 9.691656912957684e-07, "loss": -0.0196, "num_tokens": 34952999.0, "reward": 0.478125, "reward_std": 0.5006843686103821, "rewards/reward_correct/mean": 0.478125, "rewards/reward_correct/std": 0.500684380531311, "sampling/importance_sampling_ratio/max": 2.5264899134635925, "sampling/importance_sampling_ratio/mean": 0.4062116891145706, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.331268119812012, "sampling/sampling_logp_difference/mean": 0.0450795441865921, "step": 140, "step_time": 88.51336960084737 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.321875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2729.3, "completions/mean_length": 3610.175, "completions/mean_terminated_length": 1446.8697387695313, "completions/min_length": 536.4, "completions/min_terminated_length": 536.4, "entropy": 0.5646064455155283, "epoch": 0.4166666666666667, "frac_reward_zero_std": 0.225, "grad_norm": 0.2582959830760956, "learning_rate": 9.633218339615432e-07, "loss": -0.0341, "num_tokens": 37314103.0, "reward": 0.4046875, "reward_std": 0.47060964107513426, "rewards/reward_correct/mean": 0.4046875, "rewards/reward_correct/std": 0.470609650015831, "sampling/importance_sampling_ratio/max": 2.4045764088630674, "sampling/importance_sampling_ratio/mean": 0.3732295721769333, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.39520835876465, "sampling/sampling_logp_difference/mean": 0.04807058311998844, "step": 150, "step_time": 93.85687965862454 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2705.6, "completions/mean_length": 3746.8171875, "completions/mean_terminated_length": 1490.0490234375, "completions/min_length": 519.2, "completions/min_terminated_length": 519.2, "entropy": 0.5504673863761127, "epoch": 0.4444444444444444, "frac_reward_zero_std": 0.1625, "grad_norm": 0.20146092772483826, "learning_rate": 9.569920160040814e-07, "loss": -0.0387, "num_tokens": 39761090.0, "reward": 0.4515625, "reward_std": 0.48955525159835817, "rewards/reward_correct/mean": 0.4515625, "rewards/reward_correct/std": 0.48955525159835817, "sampling/importance_sampling_ratio/max": 2.39810996055603, "sampling/importance_sampling_ratio/mean": 0.35599239766597746, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.339636421203615, "sampling/sampling_logp_difference/mean": 0.04856264889240265, "step": 160, "step_time": 95.43752448409796 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3578125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2787.9, "completions/mean_length": 3883.9484375, "completions/mean_terminated_length": 1488.986962890625, "completions/min_length": 600.0, "completions/min_terminated_length": 600.0, "entropy": 0.5517090468667447, "epoch": 0.4722222222222222, "frac_reward_zero_std": 0.3625, "grad_norm": 0.23341695964336395, "learning_rate": 9.501828765283294e-07, "loss": -0.0221, "num_tokens": 42296545.0, "reward": 0.3953125, "reward_std": 0.47710200548172, "rewards/reward_correct/mean": 0.3953125, "rewards/reward_correct/std": 0.4771020084619522, "sampling/importance_sampling_ratio/max": 2.4312403798103333, "sampling/importance_sampling_ratio/mean": 0.3166383683681488, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.572778701782227, "sampling/sampling_logp_difference/mean": 0.04768793135881424, "step": 170, "step_time": 98.4165175858885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.340625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2983.3, "completions/mean_length": 3747.296875, "completions/mean_terminated_length": 1446.30537109375, "completions/min_length": 559.2, "completions/min_terminated_length": 559.2, "entropy": 0.5527338188607246, "epoch": 0.5, "frac_reward_zero_std": 0.2375, "grad_norm": 0.13981030881404877, "learning_rate": 9.429015573813162e-07, "loss": -0.0258, "num_tokens": 44746079.0, "reward": 0.390625, "reward_std": 0.48312163949012754, "rewards/reward_correct/mean": 0.390625, "rewards/reward_correct/std": 0.4831216514110565, "sampling/importance_sampling_ratio/max": 2.236375367641449, "sampling/importance_sampling_ratio/mean": 0.35382367074489596, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.376666259765624, "sampling/sampling_logp_difference/mean": 0.04669986329972744, "step": 180, "step_time": 95.65556847061961 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.31875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2660.9, "completions/mean_length": 3635.1734375, "completions/mean_terminated_length": 1505.3306030273438, "completions/min_length": 580.8, "completions/min_terminated_length": 580.8, "entropy": 0.536462911311537, "epoch": 0.5277777777777778, "frac_reward_zero_std": 0.1875, "grad_norm": 0.16308102011680603, "learning_rate": 9.351556956613422e-07, "loss": -0.018, "num_tokens": 47120230.0, "reward": 0.41875, "reward_std": 0.48690163493156435, "rewards/reward_correct/mean": 0.41875, "rewards/reward_correct/std": 0.48690164685249326, "sampling/importance_sampling_ratio/max": 2.3551482439041136, "sampling/importance_sampling_ratio/mean": 0.3526595965027809, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.034694099426268, "sampling/sampling_logp_difference/mean": 0.04380165822803974, "step": 190, "step_time": 92.7884238248691 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3421875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2682.7, "completions/mean_length": 3784.634375, "completions/mean_terminated_length": 1501.1128051757812, "completions/min_length": 506.2, "completions/min_terminated_length": 506.2, "entropy": 0.5288248831871897, "epoch": 0.5555555555555556, "frac_reward_zero_std": 0.225, "grad_norm": 0.3080485165119171, "learning_rate": 9.269534157077176e-07, "loss": -0.0415, "num_tokens": 49589284.0, "reward": 0.4328125, "reward_std": 0.49167430102825166, "rewards/reward_correct/mean": 0.4328125, "rewards/reward_correct/std": 0.4916743189096451, "sampling/importance_sampling_ratio/max": 2.2188847064971924, "sampling/importance_sampling_ratio/mean": 0.3379298195242882, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.151332664489747, "sampling/sampling_logp_difference/mean": 0.04422352984547615, "step": 200, "step_time": 96.42145730126649 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3015625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2692.1, "completions/mean_length": 3486.7875, "completions/mean_terminated_length": 1456.8675537109375, "completions/min_length": 424.7, "completions/min_terminated_length": 424.7, "entropy": 0.5138872295152396, "epoch": 0.5833333333333334, "frac_reward_zero_std": 0.2375, "grad_norm": 0.14368104934692383, "learning_rate": 9.183033205794524e-07, "loss": -0.0271, "num_tokens": 51866052.0, "reward": 0.4359375, "reward_std": 0.4865183919668198, "rewards/reward_correct/mean": 0.4359375, "rewards/reward_correct/std": 0.48651840090751647, "sampling/importance_sampling_ratio/max": 2.529734969139099, "sampling/importance_sampling_ratio/mean": 0.3310353755950928, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.605073738098145, "sampling/sampling_logp_difference/mean": 0.04353258796036243, "step": 210, "step_time": 91.41761814020575 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3109375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2680.2, "completions/mean_length": 3544.5046875, "completions/mean_terminated_length": 1457.0344970703125, "completions/min_length": 569.1, "completions/min_terminated_length": 569.1, "entropy": 0.5135231144260615, "epoch": 0.6111111111111112, "frac_reward_zero_std": 0.225, "grad_norm": 0.17278236150741577, "learning_rate": 9.092144830318357e-07, "loss": -0.0254, "num_tokens": 54182551.0, "reward": 0.43125, "reward_std": 0.4832146823406219, "rewards/reward_correct/mean": 0.43125, "rewards/reward_correct/std": 0.4832146853208542, "sampling/importance_sampling_ratio/max": 2.305103623867035, "sampling/importance_sampling_ratio/mean": 0.35843545794487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.62167282104492, "sampling/sampling_logp_difference/mean": 0.04431740902364254, "step": 220, "step_time": 92.90139632392675 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3234375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2705.8, "completions/mean_length": 3632.15, "completions/mean_terminated_length": 1470.5746459960938, "completions/min_length": 552.3, "completions/min_terminated_length": 552.3, "entropy": 0.5150754484347999, "epoch": 0.6388888888888888, "frac_reward_zero_std": 0.1875, "grad_norm": 0.4685378968715668, "learning_rate": 8.99696436000368e-07, "loss": -0.0234, "num_tokens": 56552135.0, "reward": 0.4265625, "reward_std": 0.48878895938396455, "rewards/reward_correct/mean": 0.4265625, "rewards/reward_correct/std": 0.48878897428512574, "sampling/importance_sampling_ratio/max": 2.552997076511383, "sampling/importance_sampling_ratio/mean": 0.3578918635845184, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.531307792663576, "sampling/sampling_logp_difference/mean": 0.04508708380162716, "step": 230, "step_time": 93.72880325838923 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3234375, "completions/max_length": 8192.0, "completions/max_terminated_length": 3232.5, "completions/mean_length": 3705.865625, "completions/mean_terminated_length": 1559.2656982421875, "completions/min_length": 649.0, "completions/min_terminated_length": 649.0, "entropy": 0.5137905349489301, "epoch": 0.6666666666666666, "frac_reward_zero_std": 0.2875, "grad_norm": 0.1393118053674698, "learning_rate": 8.897591626020284e-07, "loss": -0.0271, "num_tokens": 58972081.0, "reward": 0.4296875, "reward_std": 0.4902578145265579, "rewards/reward_correct/mean": 0.4296875, "rewards/reward_correct/std": 0.49025782346725466, "sampling/importance_sampling_ratio/max": 2.492326486110687, "sampling/importance_sampling_ratio/mean": 0.33982470631599426, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 19.714303398132323, "sampling/sampling_logp_difference/mean": 0.04368722774088383, "step": 240, "step_time": 94.55036701764911 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3140625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2707.3, "completions/mean_length": 3529.0140625, "completions/mean_terminated_length": 1392.2738159179687, "completions/min_length": 495.1, "completions/min_terminated_length": 495.1, "entropy": 0.5081628792919218, "epoch": 0.6944444444444444, "frac_reward_zero_std": 0.175, "grad_norm": 0.08988256007432938, "learning_rate": 8.794130856643633e-07, "loss": -0.0172, "num_tokens": 61275938.0, "reward": 0.4484375, "reward_std": 0.49697442054748536, "rewards/reward_correct/mean": 0.4484375, "rewards/reward_correct/std": 0.4969744235277176, "sampling/importance_sampling_ratio/max": 2.635455274581909, "sampling/importance_sampling_ratio/mean": 0.35613458752632143, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.6952974319458, "sampling/sampling_logp_difference/mean": 0.044385163486003874, "step": 250, "step_time": 92.77678105384112 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.275, "completions/max_length": 8192.0, "completions/max_terminated_length": 3083.6, "completions/mean_length": 3301.2484375, "completions/mean_terminated_length": 1451.84169921875, "completions/min_length": 537.7, "completions/min_terminated_length": 537.7, "entropy": 0.48980009178631007, "epoch": 0.7222222222222222, "frac_reward_zero_std": 0.175, "grad_norm": 0.13209204375743866, "learning_rate": 8.686690567933801e-07, "loss": -0.0239, "num_tokens": 63433217.0, "reward": 0.4625, "reward_std": 0.5000784158706665, "rewards/reward_correct/mean": 0.4625, "rewards/reward_correct/std": 0.5000784218311309, "sampling/importance_sampling_ratio/max": 2.4186819911003115, "sampling/importance_sampling_ratio/mean": 0.3675165086984634, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.942532539367676, "sampling/sampling_logp_difference/mean": 0.04302049204707146, "step": 260, "step_time": 88.44859280735254 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3078125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2721.3, "completions/mean_length": 3572.4546875, "completions/mean_terminated_length": 1534.3489501953125, "completions/min_length": 564.8, "completions/min_terminated_length": 564.8, "entropy": 0.5065572742838412, "epoch": 0.75, "frac_reward_zero_std": 0.2625, "grad_norm": 0.3504384756088257, "learning_rate": 8.575383449917102e-07, "loss": -0.0168, "num_tokens": 65767596.0, "reward": 0.453125, "reward_std": 0.4748325616121292, "rewards/reward_correct/mean": 0.453125, "rewards/reward_correct/std": 0.4748325675725937, "sampling/importance_sampling_ratio/max": 2.2907237648963927, "sampling/importance_sampling_ratio/mean": 0.3160614877939224, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 25.577795219421386, "sampling/sampling_logp_difference/mean": 0.043240266665816304, "step": 270, "step_time": 92.66369731239975 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2734375, "completions/max_length": 8192.0, "completions/max_terminated_length": 2857.0, "completions/mean_length": 3278.6984375, "completions/mean_terminated_length": 1444.8076782226562, "completions/min_length": 553.0, "completions/min_terminated_length": 553.0, "entropy": 0.4928085923660547, "epoch": 0.7777777777777778, "frac_reward_zero_std": 0.2, "grad_norm": 0.11743981391191483, "learning_rate": 8.460326248389824e-07, "loss": -0.0328, "num_tokens": 67909899.0, "reward": 0.4796875, "reward_std": 0.4822675257921219, "rewards/reward_correct/mean": 0.4796875, "rewards/reward_correct/std": 0.4822675287723541, "sampling/importance_sampling_ratio/max": 2.5242951631546022, "sampling/importance_sampling_ratio/mean": 0.38842698484659194, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.473637580871582, "sampling/sampling_logp_difference/mean": 0.04052827805280686, "step": 280, "step_time": 87.84348612818867 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2953125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2763.8, "completions/mean_length": 3433.0734375, "completions/mean_terminated_length": 1448.4840942382812, "completions/min_length": 560.3, "completions/min_terminated_length": 560.3, "entropy": 0.5100364525802433, "epoch": 0.8055555555555556, "frac_reward_zero_std": 0.25, "grad_norm": 0.2548004686832428, "learning_rate": 8.341639642468001e-07, "loss": -0.0231, "num_tokens": 70154122.0, "reward": 0.39375, "reward_std": 0.4714914709329605, "rewards/reward_correct/mean": 0.39375, "rewards/reward_correct/std": 0.4714914858341217, "sampling/importance_sampling_ratio/max": 2.33950617313385, "sampling/importance_sampling_ratio/mean": 0.3633014544844627, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.3909854888916, "sampling/sampling_logp_difference/mean": 0.04575198851525784, "step": 290, "step_time": 90.32039845213293 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.265625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2642.8, "completions/mean_length": 3212.525, "completions/mean_terminated_length": 1411.774169921875, "completions/min_length": 572.2, "completions/min_terminated_length": 572.2, "entropy": 0.47054290105588736, "epoch": 0.8333333333333334, "frac_reward_zero_std": 0.2125, "grad_norm": 0.27349135279655457, "learning_rate": 8.219448118011687e-07, "loss": -0.0334, "num_tokens": 72256530.0, "reward": 0.421875, "reward_std": 0.4876495271921158, "rewards/reward_correct/mean": 0.421875, "rewards/reward_correct/std": 0.48764953911304476, "sampling/importance_sampling_ratio/max": 2.2759945511817934, "sampling/importance_sampling_ratio/mean": 0.3820392429828644, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.031887817382813, "sampling/sampling_logp_difference/mean": 0.041293776035308837, "step": 300, "step_time": 86.86856737490743 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2796875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2730.5, "completions/mean_length": 3313.04375, "completions/mean_terminated_length": 1431.5855224609375, "completions/min_length": 443.0, "completions/min_terminated_length": 443.0, "entropy": 0.500835029873997, "epoch": 0.8611111111111112, "frac_reward_zero_std": 0.25, "grad_norm": 0.460864394903183, "learning_rate": 8.093879837056485e-07, "loss": -0.0167, "num_tokens": 74422630.0, "reward": 0.4078125, "reward_std": 0.4829194873571396, "rewards/reward_correct/mean": 0.4078125, "rewards/reward_correct/std": 0.4829194962978363, "sampling/importance_sampling_ratio/max": 2.56204674243927, "sampling/importance_sampling_ratio/mean": 0.3848576098680496, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.089742851257324, "sampling/sampling_logp_difference/mean": 0.04200447015464306, "step": 310, "step_time": 88.47893992438912 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.303125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2956.5, "completions/mean_length": 3513.2265625, "completions/mean_terminated_length": 1483.2241455078124, "completions/min_length": 610.5, "completions/min_terminated_length": 610.5, "entropy": 0.514991466421634, "epoch": 0.8888888888888888, "frac_reward_zero_std": 0.2625, "grad_norm": 0.179782897233963, "learning_rate": 7.965066503389264e-07, "loss": -0.0309, "num_tokens": 76716127.0, "reward": 0.365625, "reward_std": 0.47816264927387236, "rewards/reward_correct/mean": 0.365625, "rewards/reward_correct/std": 0.4781626582145691, "sampling/importance_sampling_ratio/max": 2.519623303413391, "sampling/importance_sampling_ratio/mean": 0.3532961577177048, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.083291053771973, "sampling/sampling_logp_difference/mean": 0.04559468738734722, "step": 320, "step_time": 91.2618560783565 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.253125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2747.5, "completions/mean_length": 3135.4203125, "completions/mean_terminated_length": 1434.197314453125, "completions/min_length": 520.0, "completions/min_terminated_length": 520.0, "entropy": 0.4833090817555785, "epoch": 0.9166666666666666, "frac_reward_zero_std": 0.1875, "grad_norm": 0.13914607465267181, "learning_rate": 7.833143224409075e-07, "loss": -0.0312, "num_tokens": 78771436.0, "reward": 0.4390625, "reward_std": 0.4904475510120392, "rewards/reward_correct/mean": 0.4390625, "rewards/reward_correct/std": 0.49044756293296815, "sampling/importance_sampling_ratio/max": 2.455558753013611, "sampling/importance_sampling_ratio/mean": 0.38001332730054854, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.64868927001953, "sampling/sampling_logp_difference/mean": 0.04199961461126804, "step": 330, "step_time": 85.40838945042341 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.290625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2742.6, "completions/mean_length": 3391.753125, "completions/mean_terminated_length": 1421.10107421875, "completions/min_length": 449.9, "completions/min_terminated_length": 449.9, "entropy": 0.4844976777676493, "epoch": 0.9444444444444444, "frac_reward_zero_std": 0.225, "grad_norm": 0.16815045475959778, "learning_rate": 7.698248369418146e-07, "loss": -0.0224, "num_tokens": 80986918.0, "reward": 0.4578125, "reward_std": 0.49601399302482607, "rewards/reward_correct/mean": 0.4578125, "rewards/reward_correct/std": 0.496014004945755, "sampling/importance_sampling_ratio/max": 2.3630632042884825, "sampling/importance_sampling_ratio/mean": 0.3652906835079193, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.45225372314453, "sampling/sampling_logp_difference/mean": 0.0402510404586792, "step": 340, "step_time": 89.63123638182878 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3, "completions/max_length": 8192.0, "completions/max_terminated_length": 2764.0, "completions/mean_length": 3510.5140625, "completions/mean_terminated_length": 1515.0855346679687, "completions/min_length": 589.2, "completions/min_terminated_length": 589.2, "entropy": 0.48650123639963566, "epoch": 0.9722222222222222, "frac_reward_zero_std": 0.2125, "grad_norm": 0.34350690245628357, "learning_rate": 7.560523424491594e-07, "loss": -0.0327, "num_tokens": 83279343.0, "reward": 0.40625, "reward_std": 0.48524410724639894, "rewards/reward_correct/mean": 0.40625, "rewards/reward_correct/std": 0.4852441310882568, "sampling/importance_sampling_ratio/max": 2.621637427806854, "sampling/importance_sampling_ratio/mean": 0.3800614088773727, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.8350399017334, "sampling/sampling_logp_difference/mean": 0.04253811165690422, "step": 350, "step_time": 90.70822816956789 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.315625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2796.9, "completions/mean_length": 3580.165625, "completions/mean_terminated_length": 1469.3232055664062, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "entropy": 0.5141544749494642, "epoch": 1.0, "frac_reward_zero_std": 0.2375, "grad_norm": 0.09957528114318848, "learning_rate": 7.420112844078065e-07, "loss": -0.0264, "num_tokens": 85620353.0, "reward": 0.45, "reward_std": 0.48938204646110534, "rewards/reward_correct/mean": 0.45, "rewards/reward_correct/std": 0.48938204944133756, "sampling/importance_sampling_ratio/max": 2.5152801275253296, "sampling/importance_sampling_ratio/mean": 0.3644652560353279, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.44343662261963, "sampling/sampling_logp_difference/mean": 0.043626537173986436, "step": 360, "step_time": 92.83569850176573 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.2875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2694.5, "completions/mean_length": 3396.1671875, "completions/mean_terminated_length": 1472.170263671875, "completions/min_length": 472.1, "completions/min_terminated_length": 472.1, "entropy": 0.5021241253707558, "epoch": 1.0277777777777777, "frac_reward_zero_std": 0.25, "grad_norm": 0.20708823204040527, "learning_rate": 7.277163899486974e-07, "loss": -0.0084, "num_tokens": 87839972.0, "reward": 0.4609375, "reward_std": 0.4824483096599579, "rewards/reward_correct/mean": 0.4609375, "rewards/reward_correct/std": 0.4824483096599579, "sampling/importance_sampling_ratio/max": 2.517948567867279, "sampling/importance_sampling_ratio/mean": 0.4134147673845291, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.941598892211914, "sampling/sampling_logp_difference/mean": 0.04388279486447573, "step": 370, "step_time": 89.34413652997463 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3171875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2708.5, "completions/mean_length": 3588.9296875, "completions/mean_terminated_length": 1454.8683471679688, "completions/min_length": 524.1, "completions/min_terminated_length": 524.1, "entropy": 0.502693377295509, "epoch": 1.0555555555555556, "frac_reward_zero_std": 0.275, "grad_norm": 0.1858949214220047, "learning_rate": 7.131826524421229e-07, "loss": -0.0318, "num_tokens": 90182887.0, "reward": 0.440625, "reward_std": 0.4875584036111832, "rewards/reward_correct/mean": 0.440625, "rewards/reward_correct/std": 0.4875584065914154, "sampling/importance_sampling_ratio/max": 2.4215506672859193, "sampling/importance_sampling_ratio/mean": 0.36463034301996233, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.958362197875978, "sampling/sampling_logp_difference/mean": 0.04441062249243259, "step": 380, "step_time": 92.67752550859004 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3890625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2740.1, "completions/mean_length": 4121.328125, "completions/mean_terminated_length": 1532.1543334960938, "completions/min_length": 603.0, "completions/min_terminated_length": 603.0, "entropy": 0.5237304116133601, "epoch": 1.0833333333333333, "frac_reward_zero_std": 0.25, "grad_norm": 0.09304118901491165, "learning_rate": 6.984253157717485e-07, "loss": -0.018, "num_tokens": 92870865.0, "reward": 0.4078125, "reward_std": 0.49269229769706724, "rewards/reward_correct/mean": 0.4078125, "rewards/reward_correct/std": 0.492692306637764, "sampling/importance_sampling_ratio/max": 2.14275484085083, "sampling/importance_sampling_ratio/mean": 0.29739712178707123, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.217792129516603, "sampling/sampling_logp_difference/mean": 0.04599142037332058, "step": 390, "step_time": 102.05327154491097 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3640625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2618.8, "completions/mean_length": 3917.4375, "completions/mean_terminated_length": 1481.8446899414062, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "entropy": 0.5371272402349859, "epoch": 1.1111111111111112, "frac_reward_zero_std": 0.2125, "grad_norm": 0.14799313247203827, "learning_rate": 6.834598583458861e-07, "loss": -0.0227, "num_tokens": 95421425.0, "reward": 0.428125, "reward_std": 0.482084384560585, "rewards/reward_correct/mean": 0.428125, "rewards/reward_correct/std": 0.4820843905210495, "sampling/importance_sampling_ratio/max": 2.424322748184204, "sampling/importance_sampling_ratio/mean": 0.32391665428876876, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.81523914337158, "sampling/sampling_logp_difference/mean": 0.04645326770842075, "step": 400, "step_time": 99.04672075342387 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.371875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2735.0, "completions/mean_length": 3984.6125, "completions/mean_terminated_length": 1514.8505249023438, "completions/min_length": 551.9, "completions/min_terminated_length": 551.9, "entropy": 0.5294327767100185, "epoch": 1.1388888888888888, "frac_reward_zero_std": 0.2, "grad_norm": 0.2487545609474182, "learning_rate": 6.683019768627794e-07, "loss": -0.0202, "num_tokens": 98020353.0, "reward": 0.4203125, "reward_std": 0.4853229820728302, "rewards/reward_correct/mean": 0.4203125, "rewards/reward_correct/std": 0.48532298803329466, "sampling/importance_sampling_ratio/max": 2.425787329673767, "sampling/importance_sampling_ratio/mean": 0.34082319438457487, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.107694625854492, "sampling/sampling_logp_difference/mean": 0.04505102261900902, "step": 410, "step_time": 100.24638866372406 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4046875, "completions/max_length": 8192.0, "completions/max_terminated_length": 3061.0, "completions/mean_length": 4253.1953125, "completions/mean_terminated_length": 1604.5595947265624, "completions/min_length": 520.9, "completions/min_terminated_length": 520.9, "entropy": 0.5594399727415293, "epoch": 1.1666666666666667, "frac_reward_zero_std": 0.3125, "grad_norm": 0.09589551389217377, "learning_rate": 6.529675698469369e-07, "loss": -0.0194, "num_tokens": 100791518.0, "reward": 0.340625, "reward_std": 0.4659861117601395, "rewards/reward_correct/mean": 0.340625, "rewards/reward_correct/std": 0.4659861266613007, "sampling/importance_sampling_ratio/max": 2.2143738508224486, "sampling/importance_sampling_ratio/mean": 0.2883808270096779, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 24.680903816223143, "sampling/sampling_logp_difference/mean": 0.045639732107520106, "step": 420, "step_time": 103.95541290417313 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3640625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2721.9, "completions/mean_length": 3928.7765625, "completions/mean_terminated_length": 1515.8645141601562, "completions/min_length": 658.3, "completions/min_terminated_length": 658.3, "entropy": 0.5309726831037551, "epoch": 1.1944444444444444, "frac_reward_zero_std": 0.2125, "grad_norm": 0.11377198994159698, "learning_rate": 6.374727209737742e-07, "loss": -0.0375, "num_tokens": 103356455.0, "reward": 0.3875, "reward_std": 0.4833220809698105, "rewards/reward_correct/mean": 0.3875, "rewards/reward_correct/std": 0.48332209289073946, "sampling/importance_sampling_ratio/max": 2.697994363307953, "sampling/importance_sampling_ratio/mean": 0.3253849983215332, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.634774589538573, "sampling/sampling_logp_difference/mean": 0.04580756314098835, "step": 430, "step_time": 98.64194328989834 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3828125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2776.0, "completions/mean_length": 4030.6875, "completions/mean_terminated_length": 1456.2660522460938, "completions/min_length": 534.9, "completions/min_terminated_length": 534.9, "entropy": 0.5253222627099603, "epoch": 1.2222222222222223, "frac_reward_zero_std": 0.2875, "grad_norm": 0.1834605634212494, "learning_rate": 6.218336822000597e-07, "loss": -0.0311, "num_tokens": 105985199.0, "reward": 0.371875, "reward_std": 0.4680081695318222, "rewards/reward_correct/mean": 0.371875, "rewards/reward_correct/std": 0.46800817251205445, "sampling/importance_sampling_ratio/max": 2.4830429673194887, "sampling/importance_sampling_ratio/mean": 0.3344820261001587, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 25.512220191955567, "sampling/sampling_logp_difference/mean": 0.04608744233846664, "step": 440, "step_time": 101.21402945108711 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.4046875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2800.7, "completions/mean_length": 4262.6609375, "completions/mean_terminated_length": 1580.7835083007812, "completions/min_length": 585.1, "completions/min_terminated_length": 585.1, "entropy": 0.5569739128928631, "epoch": 1.25, "frac_reward_zero_std": 0.175, "grad_norm": 0.12380044907331467, "learning_rate": 6.060668567178559e-07, "loss": -0.0332, "num_tokens": 108762526.0, "reward": 0.4375, "reward_std": 0.4926897346973419, "rewards/reward_correct/mean": 0.4375, "rewards/reward_correct/std": 0.4926897376775742, "sampling/importance_sampling_ratio/max": 2.0959500551223753, "sampling/importance_sampling_ratio/mean": 0.28630392253398895, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.07707347869873, "sampling/sampling_logp_difference/mean": 0.048703621700406076, "step": 450, "step_time": 104.36413167230785 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3203125, "completions/max_length": 8192.0, "completions/max_terminated_length": 2636.1, "completions/mean_length": 3645.103125, "completions/mean_terminated_length": 1502.7676513671875, "completions/min_length": 578.3, "completions/min_terminated_length": 578.3, "entropy": 0.5052359893452376, "epoch": 1.2777777777777777, "frac_reward_zero_std": 0.1375, "grad_norm": 0.3007439970970154, "learning_rate": 5.901887817498367e-07, "loss": -0.0368, "num_tokens": 111138808.0, "reward": 0.41875, "reward_std": 0.49026038646698, "rewards/reward_correct/mean": 0.41875, "rewards/reward_correct/std": 0.49026039242744446, "sampling/importance_sampling_ratio/max": 2.2202050924301147, "sampling/importance_sampling_ratio/mean": 0.3308444514870644, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 23.17190055847168, "sampling/sampling_logp_difference/mean": 0.04389919862151146, "step": 460, "step_time": 93.38349273055792 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3265625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2857.9, "completions/mean_length": 3689.3828125, "completions/mean_terminated_length": 1499.214501953125, "completions/min_length": 502.0, "completions/min_terminated_length": 502.0, "entropy": 0.5224037424195558, "epoch": 1.3055555555555556, "frac_reward_zero_std": 0.2, "grad_norm": 0.2663036286830902, "learning_rate": 5.742161112040236e-07, "loss": -0.0214, "num_tokens": 113549213.0, "reward": 0.375, "reward_std": 0.4806476831436157, "rewards/reward_correct/mean": 0.375, "rewards/reward_correct/std": 0.48064770102500914, "sampling/importance_sampling_ratio/max": 2.4689416170120237, "sampling/importance_sampling_ratio/mean": 0.3618432849645615, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 20.738582611083984, "sampling/sampling_logp_difference/mean": 0.046205943822860716, "step": 470, "step_time": 94.04668763801456 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.35, "completions/max_length": 8192.0, "completions/max_terminated_length": 3171.3, "completions/mean_length": 3906.228125, "completions/mean_terminated_length": 1615.4124145507812, "completions/min_length": 649.1, "completions/min_terminated_length": 649.1, "entropy": 0.5468584059271961, "epoch": 1.3333333333333333, "frac_reward_zero_std": 0.2, "grad_norm": 0.1444406360387802, "learning_rate": 5.581655982061366e-07, "loss": -0.0381, "num_tokens": 116095783.0, "reward": 0.4109375, "reward_std": 0.48344514071941375, "rewards/reward_correct/mean": 0.4109375, "rewards/reward_correct/std": 0.48344514071941375, "sampling/importance_sampling_ratio/max": 2.212104618549347, "sampling/importance_sampling_ratio/mean": 0.3308627545833588, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 22.928424453735353, "sampling/sampling_logp_difference/mean": 0.04624480046331882, "step": 480, "step_time": 97.89495770912617 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.3640625, "completions/max_length": 8192.0, "completions/max_terminated_length": 2847.0, "completions/mean_length": 3933.9484375, "completions/mean_terminated_length": 1534.0626586914063, "completions/min_length": 625.0, "completions/min_terminated_length": 625.0, "entropy": 0.5143518089316785, "epoch": 1.3611111111111112, "frac_reward_zero_std": 0.1625, "grad_norm": 0.14443330466747284, "learning_rate": 5.420540775278788e-07, "loss": -0.0338, "num_tokens": 118660662.0, "reward": 0.453125, "reward_std": 0.490170481801033, "rewards/reward_correct/mean": 0.453125, "rewards/reward_correct/std": 0.49017049074172975, "sampling/importance_sampling_ratio/max": 2.4240808367729185, "sampling/importance_sampling_ratio/mean": 0.32956405729055405, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 25.39155864715576, "sampling/sampling_logp_difference/mean": 0.044797389209270476, "step": 490, "step_time": 99.29671751111746 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.31875, "completions/max_length": 8192.0, "completions/max_terminated_length": 2790.6, "completions/mean_length": 3592.6796875, "completions/mean_terminated_length": 1455.6937377929687, "completions/min_length": 574.1, "completions/min_terminated_length": 574.1, "entropy": 0.5269695220049471, "epoch": 1.3888888888888888, "frac_reward_zero_std": 0.25, "grad_norm": 0.15532448887825012, "learning_rate": 5.258984479295852e-07, "loss": -0.028, "num_tokens": 121006129.0, "reward": 0.4578125, "reward_std": 0.4924586594104767, "rewards/reward_correct/mean": 0.4578125, "rewards/reward_correct/std": 0.49245867431163787, "sampling/importance_sampling_ratio/max": 2.5489956021308897, "sampling/importance_sampling_ratio/mean": 0.37822050750255587, "sampling/importance_sampling_ratio/min": 0.0, "sampling/sampling_logp_difference/max": 21.463522529602052, "sampling/sampling_logp_difference/mean": 0.046779343485832216, "step": 500, "step_time": 92.61194766778499 } ], "logging_steps": 10, "max_steps": 1000, "num_input_tokens_seen": 121006129, "num_train_epochs": 3, "save_steps": 100, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }