| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.1404494382022472, |
| "eval_steps": 500, |
| "global_step": 100, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.35, |
| "completions/max_length": 6342.6, |
| "completions/max_terminated_length": 4389.8, |
| "completions/mean_length": 5336.85, |
| "completions/mean_terminated_length": 3317.43818359375, |
| "completions/min_length": 3839.2, |
| "completions/min_terminated_length": 2220.0, |
| "entropy": 0.13346208892762662, |
| "epoch": 0.007022471910112359, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 9.64939933512099e-18, |
| "learning_rate": 5e-07, |
| "loss": -5.502499733292571e-18, |
| "num_tokens": 217434.0, |
| "reward": 0.23524999916553496, |
| "reward_std": 0.33813109397888186, |
| "rewards/compute_reward/mean": 0.23524999916553496, |
| "rewards/compute_reward/std": 0.33813111782073973, |
| "sampling/importance_sampling_ratio/max": 0.600124711072749, |
| "sampling/importance_sampling_ratio/mean": 0.35002241265671047, |
| "sampling/importance_sampling_ratio/min": 0.20000003157700258, |
| "sampling/sampling_logp_difference/max": 0.40465545654296875, |
| "sampling/sampling_logp_difference/mean": 0.012414481490850449, |
| "step": 5, |
| "step_time": 78.7855021980009 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 8082.8, |
| "completions/max_terminated_length": 5149.0, |
| "completions/mean_length": 6318.725, |
| "completions/mean_terminated_length": 4152.404833984375, |
| "completions/min_length": 5055.0, |
| "completions/min_terminated_length": 3438.4, |
| "entropy": 0.1376650758087635, |
| "epoch": 0.014044943820224719, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 4.5075054799781356e-14, |
| "learning_rate": 5e-07, |
| "loss": -4.503626627183621e-09, |
| "num_tokens": 474551.0, |
| "reward": 0.08315625190734863, |
| "reward_std": 0.45229817628860475, |
| "rewards/compute_reward/mean": 0.08315625190734863, |
| "rewards/compute_reward/std": 0.4522981971502304, |
| "sampling/importance_sampling_ratio/max": 1.0, |
| "sampling/importance_sampling_ratio/mean": 0.5000000178813935, |
| "sampling/importance_sampling_ratio/min": 0.2, |
| "sampling/sampling_logp_difference/max": 0.4511141300201416, |
| "sampling/sampling_logp_difference/mean": 0.01602336037904024, |
| "step": 10, |
| "step_time": 101.2872915075859 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 6691.0, |
| "completions/max_terminated_length": 6335.8, |
| "completions/mean_length": 5185.35, |
| "completions/mean_terminated_length": 4696.5650390625, |
| "completions/min_length": 3638.4, |
| "completions/min_terminated_length": 3638.4, |
| "entropy": 0.1449942274019122, |
| "epoch": 0.021067415730337078, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2945383787155151e-07, |
| "learning_rate": 5e-07, |
| "loss": -1.8636472304933705e-06, |
| "num_tokens": 686789.0, |
| "reward": 0.3246250033378601, |
| "reward_std": 0.5596808314323425, |
| "rewards/compute_reward/mean": 0.3246250033378601, |
| "rewards/compute_reward/std": 0.5596808195114136, |
| "sampling/importance_sampling_ratio/max": 0.4000196443371508, |
| "sampling/importance_sampling_ratio/mean": 0.2500068536591781, |
| "sampling/importance_sampling_ratio/min": 4.741177590247785e-12, |
| "sampling/sampling_logp_difference/max": 0.5730626344680786, |
| "sampling/sampling_logp_difference/mean": 0.01585924383252859, |
| "step": 15, |
| "step_time": 76.25853070278535 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.525, |
| "completions/max_length": 6157.0, |
| "completions/max_terminated_length": 2626.2, |
| "completions/mean_length": 5558.75, |
| "completions/mean_terminated_length": 2014.99169921875, |
| "completions/min_length": 4721.0, |
| "completions/min_terminated_length": 1498.2, |
| "entropy": 0.08328314051032067, |
| "epoch": 0.028089887640449437, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 2.2411346435546875e-05, |
| "learning_rate": 5e-07, |
| "loss": -1.7270938769797795e-06, |
| "num_tokens": 913419.0, |
| "reward": 0.011406248807907105, |
| "reward_std": 0.23478415608406067, |
| "rewards/compute_reward/mean": 0.011406248807907105, |
| "rewards/compute_reward/std": 0.23478415608406067, |
| "sampling/importance_sampling_ratio/max": 0.6000377955271687, |
| "sampling/importance_sampling_ratio/mean": 0.5250090071654995, |
| "sampling/importance_sampling_ratio/min": 0.4000000000222174, |
| "sampling/sampling_logp_difference/max": 0.26677918434143066, |
| "sampling/sampling_logp_difference/mean": 0.0077647527679800986, |
| "step": 20, |
| "step_time": 86.48750274259365 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.525, |
| "completions/max_length": 7358.0, |
| "completions/max_terminated_length": 5255.0, |
| "completions/mean_length": 6298.35, |
| "completions/mean_terminated_length": 3806.3400390625, |
| "completions/min_length": 4295.4, |
| "completions/min_terminated_length": 2676.4, |
| "entropy": 0.11463551484048366, |
| "epoch": 0.0351123595505618, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 7.334165275096893e-09, |
| "learning_rate": 5e-07, |
| "loss": -4.845223333660442e-10, |
| "num_tokens": 1168593.0, |
| "reward": -0.027125003933906554, |
| "reward_std": 0.5179957747459412, |
| "rewards/compute_reward/mean": -0.027125003933906554, |
| "rewards/compute_reward/std": 0.5179957985877991, |
| "sampling/importance_sampling_ratio/max": 0.8000000000302949, |
| "sampling/importance_sampling_ratio/mean": 0.5250000000044796, |
| "sampling/importance_sampling_ratio/min": 0.2, |
| "sampling/sampling_logp_difference/max": 0.4423918962478638, |
| "sampling/sampling_logp_difference/mean": 0.013001743517816067, |
| "step": 25, |
| "step_time": 100.9442321440205 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.375, |
| "completions/max_length": 7867.0, |
| "completions/max_terminated_length": 6072.6, |
| "completions/mean_length": 5969.625, |
| "completions/mean_terminated_length": 3844.621484375, |
| "completions/min_length": 3876.8, |
| "completions/min_terminated_length": 2283.6, |
| "entropy": 0.1684934377670288, |
| "epoch": 0.042134831460674156, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": -7.479580510505457e-10, |
| "num_tokens": 1412506.0, |
| "reward": 0.21940625309944153, |
| "reward_std": 0.38065622448921205, |
| "rewards/compute_reward/mean": 0.21940625309944153, |
| "rewards/compute_reward/std": 0.38065624237060547, |
| "sampling/importance_sampling_ratio/max": 0.8000000000015144, |
| "sampling/importance_sampling_ratio/mean": 0.3750000059606567, |
| "sampling/importance_sampling_ratio/min": 0.2, |
| "sampling/sampling_logp_difference/max": 0.46221423149108887, |
| "sampling/sampling_logp_difference/mean": 0.015006156265735626, |
| "step": 30, |
| "step_time": 92.77473875220167 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.225, |
| "completions/max_length": 5447.6, |
| "completions/max_terminated_length": 4491.6, |
| "completions/mean_length": 3979.525, |
| "completions/mean_terminated_length": 3340.66669921875, |
| "completions/min_length": 2628.0, |
| "completions/min_terminated_length": 2628.0, |
| "entropy": 0.15426228623837232, |
| "epoch": 0.04915730337078652, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": 5.5907651130837624e-08, |
| "num_tokens": 1577087.0, |
| "reward": 0.2894999995827675, |
| "reward_std": 0.4346048951148987, |
| "rewards/compute_reward/mean": 0.2894999995827675, |
| "rewards/compute_reward/std": 0.43460490703582766, |
| "sampling/importance_sampling_ratio/max": 0.4101849972048058, |
| "sampling/importance_sampling_ratio/mean": 0.2273602893020211, |
| "sampling/importance_sampling_ratio/min": 9.34835527009561e-05, |
| "sampling/sampling_logp_difference/max": 0.5941443920135498, |
| "sampling/sampling_logp_difference/mean": 0.014553416334092618, |
| "step": 35, |
| "step_time": 61.56992885797517 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.7, |
| "completions/max_length": 8080.2, |
| "completions/max_terminated_length": 4724.0, |
| "completions/mean_length": 7451.025, |
| "completions/mean_terminated_length": 4068.96435546875, |
| "completions/min_length": 6761.0, |
| "completions/min_terminated_length": 3524.6, |
| "entropy": 0.06469594687223434, |
| "epoch": 0.056179775280898875, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 1.546140993013978e-11, |
| "learning_rate": 5e-07, |
| "loss": 1.2250055136941995e-12, |
| "num_tokens": 1879600.0, |
| "reward": -0.5208124965429306, |
| "reward_std": 0.28857092424295844, |
| "rewards/compute_reward/mean": -0.5208124965429306, |
| "rewards/compute_reward/std": 0.2885709242895246, |
| "sampling/importance_sampling_ratio/max": 1.0, |
| "sampling/importance_sampling_ratio/mean": 0.7, |
| "sampling/importance_sampling_ratio/min": 0.4, |
| "sampling/sampling_logp_difference/max": 0.2912771701812744, |
| "sampling/sampling_logp_difference/mean": 0.010922013968229293, |
| "step": 40, |
| "step_time": 122.65326870220596 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.475, |
| "completions/max_length": 6348.2, |
| "completions/max_terminated_length": 3134.0, |
| "completions/mean_length": 5377.575, |
| "completions/mean_terminated_length": 1950.0400390625, |
| "completions/min_length": 4405.0, |
| "completions/min_terminated_length": 1193.0, |
| "entropy": 0.12274600118398667, |
| "epoch": 0.06320224719101124, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": 5.91442221775651e-05, |
| "num_tokens": 2098759.0, |
| "reward": -0.26150000691413877, |
| "reward_std": 0.26170684695243834, |
| "rewards/compute_reward/mean": -0.26150000691413877, |
| "rewards/compute_reward/std": 0.2617068350315094, |
| "sampling/importance_sampling_ratio/max": 0.6020115901934332, |
| "sampling/importance_sampling_ratio/mean": 0.47526856281074287, |
| "sampling/importance_sampling_ratio/min": 0.4000000296924043, |
| "sampling/sampling_logp_difference/max": 0.2926159858703613, |
| "sampling/sampling_logp_difference/mean": 0.010471996665000916, |
| "step": 45, |
| "step_time": 85.52473701598937 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.125, |
| "completions/max_length": 7601.6, |
| "completions/max_terminated_length": 6804.0, |
| "completions/mean_length": 5256.25, |
| "completions/mean_terminated_length": 4877.31513671875, |
| "completions/min_length": 3520.8, |
| "completions/min_terminated_length": 3520.8, |
| "entropy": 0.1676523458212614, |
| "epoch": 0.0702247191011236, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": -8.948970275923785e-09, |
| "num_tokens": 2313249.0, |
| "reward": 0.6231875061988831, |
| "reward_std": 0.30267552062869074, |
| "rewards/compute_reward/mean": 0.6231875061988831, |
| "rewards/compute_reward/std": 0.302675536647439, |
| "sampling/importance_sampling_ratio/max": 0.6000000545625805, |
| "sampling/importance_sampling_ratio/mean": 0.12500003689434863, |
| "sampling/importance_sampling_ratio/min": 1.509659897099789e-11, |
| "sampling/sampling_logp_difference/max": 0.6264592289924622, |
| "sampling/sampling_logp_difference/mean": 0.014229525998234748, |
| "step": 50, |
| "step_time": 76.5796675938298 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.6, |
| "completions/max_length": 7477.6, |
| "completions/max_terminated_length": 5252.6, |
| "completions/mean_length": 6872.8, |
| "completions/mean_terminated_length": 4782.11669921875, |
| "completions/min_length": 6005.6, |
| "completions/min_terminated_length": 4387.0, |
| "entropy": 0.12369837798178196, |
| "epoch": 0.07724719101123595, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": -2.9906460688816355e-19, |
| "num_tokens": 2592889.0, |
| "reward": -0.4731250017881393, |
| "reward_std": 0.21400290131568908, |
| "rewards/compute_reward/mean": -0.4731250017881393, |
| "rewards/compute_reward/std": 0.21400290429592134, |
| "sampling/importance_sampling_ratio/max": 0.8000000000400606, |
| "sampling/importance_sampling_ratio/mean": 0.6000000000050764, |
| "sampling/importance_sampling_ratio/min": 0.2, |
| "sampling/sampling_logp_difference/max": 0.37660536766052244, |
| "sampling/sampling_logp_difference/mean": 0.01498084533959627, |
| "step": 55, |
| "step_time": 109.11029338700465 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.3, |
| "completions/max_length": 6866.0, |
| "completions/max_terminated_length": 6220.2, |
| "completions/mean_length": 5375.525, |
| "completions/mean_terminated_length": 4569.51435546875, |
| "completions/min_length": 3334.8, |
| "completions/min_terminated_length": 3334.8, |
| "entropy": 0.14546145126223564, |
| "epoch": 0.08426966292134831, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 8.715250743307479e-15, |
| "learning_rate": 5e-07, |
| "loss": -0.0002411100547760725, |
| "num_tokens": 2811574.0, |
| "reward": 0.1729999989271164, |
| "reward_std": 0.6171267032623291, |
| "rewards/compute_reward/mean": 0.1729999989271164, |
| "rewards/compute_reward/std": 0.6171267211437226, |
| "sampling/importance_sampling_ratio/max": 0.8053813435137271, |
| "sampling/importance_sampling_ratio/mean": 0.30101747438311577, |
| "sampling/importance_sampling_ratio/min": 3.9959536479727876e-08, |
| "sampling/sampling_logp_difference/max": 0.5940536499023438, |
| "sampling/sampling_logp_difference/mean": 0.015674336068332195, |
| "step": 60, |
| "step_time": 82.90924808238633 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.25, |
| "completions/max_length": 6714.4, |
| "completions/max_terminated_length": 5924.6, |
| "completions/mean_length": 4677.0, |
| "completions/mean_terminated_length": 4329.3583984375, |
| "completions/min_length": 3018.8, |
| "completions/min_terminated_length": 3018.8, |
| "entropy": 0.15218620076775552, |
| "epoch": 0.09129213483146068, |
| "frac_reward_zero_std": 0.4, |
| "grad_norm": 7.748603820800781e-07, |
| "learning_rate": 5e-07, |
| "loss": -1.951258497001618e-08, |
| "num_tokens": 3004854.0, |
| "reward": 0.4383750021457672, |
| "reward_std": 0.43163261413574217, |
| "rewards/compute_reward/mean": 0.4383750021457672, |
| "rewards/compute_reward/std": 0.43163261413574217, |
| "sampling/importance_sampling_ratio/max": 0.6001360186763123, |
| "sampling/importance_sampling_ratio/mean": 0.2500173559734094, |
| "sampling/importance_sampling_ratio/min": 8.312385355119191e-16, |
| "sampling/sampling_logp_difference/max": 0.5871267795562745, |
| "sampling/sampling_logp_difference/mean": 0.014853741601109504, |
| "step": 65, |
| "step_time": 72.67464896581369 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.5, |
| "completions/max_length": 7633.4, |
| "completions/max_terminated_length": 5260.2, |
| "completions/mean_length": 6409.45, |
| "completions/mean_terminated_length": 4188.1984375, |
| "completions/min_length": 4947.8, |
| "completions/min_terminated_length": 3327.2, |
| "entropy": 0.10757671780884266, |
| "epoch": 0.09831460674157304, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": -2.9429700343364117e-11, |
| "num_tokens": 3265376.0, |
| "reward": 0.09690623879432678, |
| "reward_std": 0.5412219107151032, |
| "rewards/compute_reward/mean": 0.09690623879432678, |
| "rewards/compute_reward/std": 0.54122194647789, |
| "sampling/importance_sampling_ratio/max": 0.8000000000121613, |
| "sampling/importance_sampling_ratio/mean": 0.5000000000016139, |
| "sampling/importance_sampling_ratio/min": 0.2, |
| "sampling/sampling_logp_difference/max": 0.5057941436767578, |
| "sampling/sampling_logp_difference/mean": 0.01242091991007328, |
| "step": 70, |
| "step_time": 100.18304894539179 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.475, |
| "completions/max_length": 7485.6, |
| "completions/max_terminated_length": 4652.0, |
| "completions/mean_length": 6131.35, |
| "completions/mean_terminated_length": 3532.04501953125, |
| "completions/min_length": 4299.8, |
| "completions/min_terminated_length": 2692.0, |
| "entropy": 0.12216376923024655, |
| "epoch": 0.10533707865168539, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 1.7136335372924805e-07, |
| "learning_rate": 5e-07, |
| "loss": -1.2361371659608266e-08, |
| "num_tokens": 3514990.0, |
| "reward": 0.02084375321865082, |
| "reward_std": 0.6247592151165009, |
| "rewards/compute_reward/mean": 0.02084375321865082, |
| "rewards/compute_reward/std": 0.6247592031955719, |
| "sampling/importance_sampling_ratio/max": 0.8000003093028454, |
| "sampling/importance_sampling_ratio/mean": 0.475000045385957, |
| "sampling/importance_sampling_ratio/min": 0.20000000000001467, |
| "sampling/sampling_logp_difference/max": 0.46574991941452026, |
| "sampling/sampling_logp_difference/mean": 0.013754993863403796, |
| "step": 75, |
| "step_time": 96.75799002298155 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.2, |
| "completions/max_length": 5894.8, |
| "completions/max_terminated_length": 5214.2, |
| "completions/mean_length": 4056.425, |
| "completions/mean_terminated_length": 3393.43935546875, |
| "completions/min_length": 2331.4, |
| "completions/min_terminated_length": 2331.4, |
| "entropy": 0.14181112498044968, |
| "epoch": 0.11235955056179775, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.00015544891357421875, |
| "learning_rate": 5e-07, |
| "loss": 4.291571385692805e-05, |
| "num_tokens": 3682831.0, |
| "reward": 0.1677812457084656, |
| "reward_std": 0.41842788457870483, |
| "rewards/compute_reward/mean": 0.1677812457084656, |
| "rewards/compute_reward/std": 0.4184278786182404, |
| "sampling/importance_sampling_ratio/max": 0.4010849003840121, |
| "sampling/importance_sampling_ratio/mean": 0.20015338080880837, |
| "sampling/importance_sampling_ratio/min": 6.377647438378011e-10, |
| "sampling/sampling_logp_difference/max": 0.5928982496261597, |
| "sampling/sampling_logp_difference/mean": 0.01402966920286417, |
| "step": 80, |
| "step_time": 62.0829234021774 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.725, |
| "completions/max_length": 8033.8, |
| "completions/max_terminated_length": 5586.4, |
| "completions/mean_length": 7458.45, |
| "completions/mean_terminated_length": 5082.3, |
| "completions/min_length": 6264.2, |
| "completions/min_terminated_length": 4664.6, |
| "entropy": 0.049185825511813164, |
| "epoch": 0.11938202247191011, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": -1.0684439999408823e-09, |
| "num_tokens": 3987497.0, |
| "reward": -0.36359376907348634, |
| "reward_std": 0.35454786419868467, |
| "rewards/compute_reward/mean": -0.36359376907348634, |
| "rewards/compute_reward/std": 0.35454787611961364, |
| "sampling/importance_sampling_ratio/max": 1.0, |
| "sampling/importance_sampling_ratio/mean": 0.725, |
| "sampling/importance_sampling_ratio/min": 0.2000000000015946, |
| "sampling/sampling_logp_difference/max": 0.4322138071060181, |
| "sampling/sampling_logp_difference/mean": 0.011536732874810695, |
| "step": 85, |
| "step_time": 121.9961906942015 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.7, |
| "completions/max_length": 8083.6, |
| "completions/max_terminated_length": 5480.0, |
| "completions/mean_length": 7200.725, |
| "completions/mean_terminated_length": 4447.6, |
| "completions/min_length": 5399.0, |
| "completions/min_terminated_length": 3780.4, |
| "entropy": 0.06885569505393505, |
| "epoch": 0.12640449438202248, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 6.502887117676437e-11, |
| "learning_rate": 5e-07, |
| "loss": 3.4291680606202136e-12, |
| "num_tokens": 4279862.0, |
| "reward": -0.3385625034570694, |
| "reward_std": 0.6242223262786866, |
| "rewards/compute_reward/mean": -0.3385625034570694, |
| "rewards/compute_reward/std": 0.6242223501205444, |
| "sampling/importance_sampling_ratio/max": 1.0, |
| "sampling/importance_sampling_ratio/mean": 0.7, |
| "sampling/importance_sampling_ratio/min": 0.20000000000054166, |
| "sampling/sampling_logp_difference/max": 0.36994409561157227, |
| "sampling/sampling_logp_difference/mean": 0.012296585179865361, |
| "step": 90, |
| "step_time": 117.92537313302746 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.4, |
| "completions/max_length": 6249.8, |
| "completions/max_terminated_length": 4446.6, |
| "completions/mean_length": 5328.1, |
| "completions/mean_terminated_length": 3106.85, |
| "completions/min_length": 3716.6, |
| "completions/min_terminated_length": 2106.2, |
| "entropy": 0.11026849336922169, |
| "epoch": 0.13342696629213482, |
| "frac_reward_zero_std": 0.6, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": -5.5089893757909183e-11, |
| "num_tokens": 4497202.0, |
| "reward": 0.19196875393390656, |
| "reward_std": 0.35176035165786745, |
| "rewards/compute_reward/mean": 0.19196875393390656, |
| "rewards/compute_reward/std": 0.35176035165786745, |
| "sampling/importance_sampling_ratio/max": 0.603415791407131, |
| "sampling/importance_sampling_ratio/mean": 0.4010292052433563, |
| "sampling/importance_sampling_ratio/min": 0.20000481952592963, |
| "sampling/sampling_logp_difference/max": 0.38380826711654664, |
| "sampling/sampling_logp_difference/mean": 0.010995614714920521, |
| "step": 95, |
| "step_time": 80.9261963216064 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.825, |
| "completions/max_length": 8078.4, |
| "completions/max_terminated_length": 2545.0, |
| "completions/mean_length": 7691.95, |
| "completions/mean_terminated_length": 2276.2333984375, |
| "completions/min_length": 6652.6, |
| "completions/min_terminated_length": 1790.2, |
| "entropy": 0.04812453463673592, |
| "epoch": 0.1404494382022472, |
| "frac_reward_zero_std": 0.2, |
| "grad_norm": 0.0, |
| "learning_rate": 5e-07, |
| "loss": 1.299868734295816e-19, |
| "num_tokens": 4809424.0, |
| "reward": -0.5054375052452087, |
| "reward_std": 0.3607128977775574, |
| "rewards/compute_reward/mean": -0.5054375052452087, |
| "rewards/compute_reward/std": 0.36071292757987977, |
| "sampling/importance_sampling_ratio/max": 1.0, |
| "sampling/importance_sampling_ratio/mean": 0.825, |
| "sampling/importance_sampling_ratio/min": 0.6, |
| "sampling/sampling_logp_difference/max": 0.1978625774383545, |
| "sampling/sampling_logp_difference/mean": 0.0077566314488649365, |
| "step": 100, |
| "step_time": 127.10719606040512 |
| } |
| ], |
| "logging_steps": 5, |
| "max_steps": 712, |
| "num_input_tokens_seen": 4809424, |
| "num_train_epochs": 1, |
| "save_steps": 100, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
