| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9945822640433419, |
| "eval_steps": 500, |
| "global_step": 109, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 964.0, |
| "completions/max_terminated_length": 964.0, |
| "completions/mean_length": 527.6614990234375, |
| "completions/mean_terminated_length": 527.6614990234375, |
| "completions/min_length": 293.0, |
| "completions/min_terminated_length": 293.0, |
| "entropy": 0.1096649169921875, |
| "epoch": 0.009124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.023836389183998108, |
| "kl": 0.00021350383758544922, |
| "learning_rate": 2e-05, |
| "loss": -0.0088, |
| "num_tokens": 411230.0, |
| "reward": 1.1042730808258057, |
| "reward_std": 0.233424112200737, |
| "rewards/cross_solution_unittest_reward/mean": 0.481730118393898, |
| "rewards/cross_solution_unittest_reward/std": 0.22199106961488724, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 1 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 920.0, |
| "completions/max_terminated_length": 920.0, |
| "completions/mean_length": 491.33856201171875, |
| "completions/mean_terminated_length": 491.33856201171875, |
| "completions/min_length": 261.0, |
| "completions/min_terminated_length": 261.0, |
| "entropy": 0.1144561767578125, |
| "epoch": 0.018249215854006275, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029492009431123734, |
| "kl": 0.0002351701259613037, |
| "learning_rate": 1.9995921928281893e-05, |
| "loss": 0.0028, |
| "num_tokens": 821440.0, |
| "reward": 0.8423246741294861, |
| "reward_std": 0.2633346915245056, |
| "rewards/cross_solution_unittest_reward/mean": 0.371605783700943, |
| "rewards/cross_solution_unittest_reward/std": 0.25612012296915054, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 2 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1480.0, |
| "completions/max_terminated_length": 1480.0, |
| "completions/mean_length": 573.1614990234375, |
| "completions/mean_terminated_length": 573.1614990234375, |
| "completions/min_length": 295.0, |
| "completions/min_terminated_length": 295.0, |
| "entropy": 0.0967254638671875, |
| "epoch": 0.02737382378100941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.026458488777279854, |
| "kl": 0.00021409988403320312, |
| "learning_rate": 1.9983691039261358e-05, |
| "loss": -0.0114, |
| "num_tokens": 1262302.0, |
| "reward": 0.9348706603050232, |
| "reward_std": 0.2640388309955597, |
| "rewards/cross_solution_unittest_reward/mean": 0.40714406967163086, |
| "rewards/cross_solution_unittest_reward/std": 0.22153814882040024, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 3 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 871.0, |
| "completions/max_terminated_length": 871.0, |
| "completions/mean_length": 506.05731201171875, |
| "completions/mean_terminated_length": 506.05731201171875, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.12152099609375, |
| "epoch": 0.03649843170801255, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028014499694108963, |
| "kl": 0.0002561211585998535, |
| "learning_rate": 1.9963317308626916e-05, |
| "loss": -0.0048, |
| "num_tokens": 1703844.0, |
| "reward": 0.6856307983398438, |
| "reward_std": 0.24917131662368774, |
| "rewards/cross_solution_unittest_reward/mean": 0.3077041059732437, |
| "rewards/cross_solution_unittest_reward/std": 0.18602629750967026, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 4 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1224.0, |
| "completions/max_terminated_length": 1224.0, |
| "completions/mean_length": 483.484375, |
| "completions/mean_terminated_length": 483.484375, |
| "completions/min_length": 247.0, |
| "completions/min_terminated_length": 247.0, |
| "entropy": 0.1254425048828125, |
| "epoch": 0.04562303963501568, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03355209901928902, |
| "kl": 0.0002625584602355957, |
| "learning_rate": 1.99348173534855e-05, |
| "loss": -0.004, |
| "num_tokens": 2084126.0, |
| "reward": 1.038306474685669, |
| "reward_std": 0.295562207698822, |
| "rewards/cross_solution_unittest_reward/mean": 0.4519632160663605, |
| "rewards/cross_solution_unittest_reward/std": 0.27628327906131744, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 5 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1174.0, |
| "completions/mean_length": 545.671875, |
| "completions/mean_terminated_length": 529.764404296875, |
| "completions/min_length": 293.0, |
| "completions/min_terminated_length": 293.0, |
| "entropy": 0.11370849609375, |
| "epoch": 0.05474764756201882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027173170819878578, |
| "kl": 0.00024318695068359375, |
| "learning_rate": 1.989821441880933e-05, |
| "loss": 0.0171, |
| "num_tokens": 2507912.0, |
| "reward": 0.9802842140197754, |
| "reward_std": 0.19230906665325165, |
| "rewards/cross_solution_unittest_reward/mean": 0.4297175034880638, |
| "rewards/cross_solution_unittest_reward/std": 0.24670374393463135, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 6 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1065.0, |
| "completions/mean_length": 556.296875, |
| "completions/mean_terminated_length": 532.4566650390625, |
| "completions/min_length": 259.0, |
| "completions/min_terminated_length": 259.0, |
| "entropy": 0.1289520263671875, |
| "epoch": 0.06387225548902195, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02603895403444767, |
| "kl": 0.0002505183219909668, |
| "learning_rate": 1.9853538358476933e-05, |
| "loss": -0.0028, |
| "num_tokens": 2962034.0, |
| "reward": 0.9045166969299316, |
| "reward_std": 0.2844803035259247, |
| "rewards/cross_solution_unittest_reward/mean": 0.39844612032175064, |
| "rewards/cross_solution_unittest_reward/std": 0.21336368471384048, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 7 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1615.0, |
| "completions/mean_length": 534.5911865234375, |
| "completions/mean_terminated_length": 526.6292724609375, |
| "completions/min_length": 260.0, |
| "completions/min_terminated_length": 260.0, |
| "entropy": 0.111663818359375, |
| "epoch": 0.0729968634160251, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02666051872074604, |
| "kl": 0.00025323033332824707, |
| "learning_rate": 1.9800825610923937e-05, |
| "loss": -0.0147, |
| "num_tokens": 3421973.0, |
| "reward": 0.8588270545005798, |
| "reward_std": 0.2657862901687622, |
| "rewards/cross_solution_unittest_reward/mean": 0.37082912772893906, |
| "rewards/cross_solution_unittest_reward/std": 0.2715388759970665, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 8 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1115.0, |
| "completions/max_terminated_length": 1115.0, |
| "completions/mean_length": 489.3724060058594, |
| "completions/mean_terminated_length": 489.3724060058594, |
| "completions/min_length": 249.0, |
| "completions/min_terminated_length": 249.0, |
| "entropy": 0.122955322265625, |
| "epoch": 0.08212147134302823, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02615615725517273, |
| "kl": 0.0002747476100921631, |
| "learning_rate": 1.9740119169423337e-05, |
| "loss": 0.0001, |
| "num_tokens": 3813332.0, |
| "reward": 0.886775016784668, |
| "reward_std": 0.2901659309864044, |
| "rewards/cross_solution_unittest_reward/mean": 0.39264198392629623, |
| "rewards/cross_solution_unittest_reward/std": 0.22153189778327942, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 9 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 908.0, |
| "completions/max_terminated_length": 908.0, |
| "completions/mean_length": 511.34375, |
| "completions/mean_terminated_length": 511.34375, |
| "completions/min_length": 204.0, |
| "completions/min_terminated_length": 204.0, |
| "entropy": 0.1106414794921875, |
| "epoch": 0.09124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027914274483919144, |
| "kl": 0.0002880394458770752, |
| "learning_rate": 1.9671468547019575e-05, |
| "loss": 0.0019, |
| "num_tokens": 4259928.0, |
| "reward": 0.8995641469955444, |
| "reward_std": 0.2562140226364136, |
| "rewards/cross_solution_unittest_reward/mean": 0.395971842110157, |
| "rewards/cross_solution_unittest_reward/std": 0.23649413138628006, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1339.0, |
| "completions/max_terminated_length": 1339.0, |
| "completions/mean_length": 548.9244995117188, |
| "completions/mean_terminated_length": 548.9244995117188, |
| "completions/min_length": 258.0, |
| "completions/min_terminated_length": 258.0, |
| "entropy": 0.1057586669921875, |
| "epoch": 0.1003706871970345, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028584128245711327, |
| "kl": 0.0003082454204559326, |
| "learning_rate": 1.9594929736144978e-05, |
| "loss": 0.0015, |
| "num_tokens": 4686539.0, |
| "reward": 0.9073726534843445, |
| "reward_std": 0.2739028036594391, |
| "rewards/cross_solution_unittest_reward/mean": 0.40253733843564987, |
| "rewards/cross_solution_unittest_reward/std": 0.2085171639919281, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 11 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1507.0, |
| "completions/max_terminated_length": 1507.0, |
| "completions/mean_length": 504.3645935058594, |
| "completions/mean_terminated_length": 504.3645935058594, |
| "completions/min_length": 263.0, |
| "completions/min_terminated_length": 263.0, |
| "entropy": 0.1241607666015625, |
| "epoch": 0.10949529512403763, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03190146014094353, |
| "kl": 0.00035694241523742676, |
| "learning_rate": 1.9510565162951538e-05, |
| "loss": -0.0026, |
| "num_tokens": 5118263.0, |
| "reward": 0.7852699160575867, |
| "reward_std": 0.29856932163238525, |
| "rewards/cross_solution_unittest_reward/mean": 0.3413052186369896, |
| "rewards/cross_solution_unittest_reward/std": 0.2405463457107544, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 12 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1108.0, |
| "completions/mean_length": 537.6640625, |
| "completions/mean_terminated_length": 529.710205078125, |
| "completions/min_length": 237.0, |
| "completions/min_terminated_length": 237.0, |
| "entropy": 0.1090240478515625, |
| "epoch": 0.11861990305104078, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03350791707634926, |
| "kl": 0.0005662441253662109, |
| "learning_rate": 1.941844363639525e-05, |
| "loss": -0.0074, |
| "num_tokens": 5532270.0, |
| "reward": 0.9594830870628357, |
| "reward_std": 0.22333046793937683, |
| "rewards/cross_solution_unittest_reward/mean": 0.42083391547203064, |
| "rewards/cross_solution_unittest_reward/std": 0.24756141006946564, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 13 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2111.0, |
| "completions/max_terminated_length": 2111.0, |
| "completions/mean_length": 610.9375, |
| "completions/mean_terminated_length": 610.9375, |
| "completions/min_length": 245.0, |
| "completions/min_terminated_length": 245.0, |
| "entropy": 0.10589599609375, |
| "epoch": 0.1277445109780439, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028112368658185005, |
| "kl": 0.00034502148628234863, |
| "learning_rate": 1.9318640292114526e-05, |
| "loss": -0.0104, |
| "num_tokens": 5998974.0, |
| "reward": 1.0141568183898926, |
| "reward_std": 0.2032373547554016, |
| "rewards/cross_solution_unittest_reward/mean": 0.44281716644763947, |
| "rewards/cross_solution_unittest_reward/std": 0.2426743358373642, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 14 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 858.0, |
| "completions/max_terminated_length": 858.0, |
| "completions/mean_length": 463.62762451171875, |
| "completions/mean_terminated_length": 463.62762451171875, |
| "completions/min_length": 226.0, |
| "completions/min_terminated_length": 226.0, |
| "entropy": 0.12762451171875, |
| "epoch": 0.13686911890504705, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029718179255723953, |
| "kl": 0.00044339895248413086, |
| "learning_rate": 1.92112365311485e-05, |
| "loss": 0.0141, |
| "num_tokens": 6343095.0, |
| "reward": 1.0152561664581299, |
| "reward_std": 0.24558740854263306, |
| "rewards/cross_solution_unittest_reward/mean": 0.4407452195882797, |
| "rewards/cross_solution_unittest_reward/std": 0.2384948655962944, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 15 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 3444.0, |
| "completions/mean_length": 578.6536865234375, |
| "completions/mean_terminated_length": 554.989501953125, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 256.0, |
| "entropy": 0.1071319580078125, |
| "epoch": 0.1459937268320502, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028195451945066452, |
| "kl": 0.00040775537490844727, |
| "learning_rate": 1.9096319953545186e-05, |
| "loss": 0.0066, |
| "num_tokens": 6764442.0, |
| "reward": 0.8980576992034912, |
| "reward_std": 0.28982824087142944, |
| "rewards/cross_solution_unittest_reward/mean": 0.3970365896821022, |
| "rewards/cross_solution_unittest_reward/std": 0.2071005031466484, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 16 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 2372.0, |
| "completions/mean_length": 528.796875, |
| "completions/mean_terminated_length": 520.81982421875, |
| "completions/min_length": 235.0, |
| "completions/min_terminated_length": 235.0, |
| "entropy": 0.11444091796875, |
| "epoch": 0.15511833475905332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030508190393447876, |
| "kl": 0.00042378902435302734, |
| "learning_rate": 1.8973984286913584e-05, |
| "loss": 0.0154, |
| "num_tokens": 7181420.0, |
| "reward": 1.1027473211288452, |
| "reward_std": 0.28832367062568665, |
| "rewards/cross_solution_unittest_reward/mean": 0.46901851892471313, |
| "rewards/cross_solution_unittest_reward/std": 0.2841140776872635, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 17 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1141.0, |
| "completions/max_terminated_length": 1141.0, |
| "completions/mean_length": 544.4661865234375, |
| "completions/mean_terminated_length": 544.4661865234375, |
| "completions/min_length": 251.0, |
| "completions/min_terminated_length": 251.0, |
| "entropy": 0.104888916015625, |
| "epoch": 0.16424294268605646, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028670065104961395, |
| "kl": 0.00043770670890808105, |
| "learning_rate": 1.8844329309978146e-05, |
| "loss": 0.002, |
| "num_tokens": 7620935.0, |
| "reward": 0.9763619303703308, |
| "reward_std": 0.243991419672966, |
| "rewards/cross_solution_unittest_reward/mean": 0.4205486625432968, |
| "rewards/cross_solution_unittest_reward/std": 0.23177453130483627, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 18 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1056.0, |
| "completions/max_terminated_length": 1056.0, |
| "completions/mean_length": 503.703125, |
| "completions/mean_terminated_length": 503.703125, |
| "completions/min_length": 276.0, |
| "completions/min_terminated_length": 276.0, |
| "entropy": 0.113372802734375, |
| "epoch": 0.17336755061305958, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02729717455804348, |
| "kl": 0.0005033016204833984, |
| "learning_rate": 1.8707460771197773e-05, |
| "loss": 0.0014, |
| "num_tokens": 8032845.0, |
| "reward": 0.9106826782226562, |
| "reward_std": 0.27397438883781433, |
| "rewards/cross_solution_unittest_reward/mean": 0.39500948786735535, |
| "rewards/cross_solution_unittest_reward/std": 0.2559768706560135, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 960.0, |
| "completions/max_terminated_length": 960.0, |
| "completions/mean_length": 545.7396240234375, |
| "completions/mean_terminated_length": 545.7396240234375, |
| "completions/min_length": 322.0, |
| "completions/min_terminated_length": 322.0, |
| "entropy": 0.1036376953125, |
| "epoch": 0.18249215854006273, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027323100715875626, |
| "kl": 0.0005331039428710938, |
| "learning_rate": 1.856349030251589e-05, |
| "loss": 0.0049, |
| "num_tokens": 8492777.0, |
| "reward": 0.9297716021537781, |
| "reward_std": 0.29434794187545776, |
| "rewards/cross_solution_unittest_reward/mean": 0.4003482013940811, |
| "rewards/cross_solution_unittest_reward/std": 0.2353406548500061, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 20 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1354.0, |
| "completions/max_terminated_length": 1354.0, |
| "completions/mean_length": 492.17449951171875, |
| "completions/mean_terminated_length": 492.17449951171875, |
| "completions/min_length": 244.0, |
| "completions/min_terminated_length": 244.0, |
| "entropy": 0.105987548828125, |
| "epoch": 0.19161676646706588, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02947384864091873, |
| "kl": 0.0005356073379516602, |
| "learning_rate": 1.8412535328311813e-05, |
| "loss": 0.006, |
| "num_tokens": 8880476.0, |
| "reward": 0.9555484652519226, |
| "reward_std": 0.23427900671958923, |
| "rewards/cross_solution_unittest_reward/mean": 0.4216877445578575, |
| "rewards/cross_solution_unittest_reward/std": 0.21247681975364685, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 21 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1018.0, |
| "completions/max_terminated_length": 1018.0, |
| "completions/mean_length": 506.7552185058594, |
| "completions/mean_terminated_length": 506.7552185058594, |
| "completions/min_length": 218.0, |
| "completions/min_terminated_length": 218.0, |
| "entropy": 0.1118011474609375, |
| "epoch": 0.200741374394069, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03192589804530144, |
| "kl": 0.0005907416343688965, |
| "learning_rate": 1.825471896962774e-05, |
| "loss": -0.0137, |
| "num_tokens": 9256446.0, |
| "reward": 0.9712496995925903, |
| "reward_std": 0.2654087245464325, |
| "rewards/cross_solution_unittest_reward/mean": 0.41771475970745087, |
| "rewards/cross_solution_unittest_reward/std": 0.2567882165312767, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 22 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1568.0, |
| "completions/max_terminated_length": 1568.0, |
| "completions/mean_length": 527.6666870117188, |
| "completions/mean_terminated_length": 527.6666870117188, |
| "completions/min_length": 289.0, |
| "completions/min_terminated_length": 289.0, |
| "entropy": 0.108154296875, |
| "epoch": 0.20986598232107215, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02674136683344841, |
| "kl": 0.000629425048828125, |
| "learning_rate": 1.8090169943749477e-05, |
| "loss": -0.0106, |
| "num_tokens": 9710614.0, |
| "reward": 1.0353918075561523, |
| "reward_std": 0.22365829348564148, |
| "rewards/cross_solution_unittest_reward/mean": 0.4482730031013489, |
| "rewards/cross_solution_unittest_reward/std": 0.2228257954120636, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 834.0, |
| "completions/max_terminated_length": 834.0, |
| "completions/mean_length": 509.2395935058594, |
| "completions/mean_terminated_length": 509.2395935058594, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.1023101806640625, |
| "epoch": 0.21899059024807527, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02758539468050003, |
| "kl": 0.0006355643272399902, |
| "learning_rate": 1.7919022459222754e-05, |
| "loss": 0.0141, |
| "num_tokens": 10133834.0, |
| "reward": 0.8483362197875977, |
| "reward_std": 0.2694234549999237, |
| "rewards/cross_solution_unittest_reward/mean": 0.3766419366002083, |
| "rewards/cross_solution_unittest_reward/std": 0.26132430136203766, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 980.0, |
| "completions/mean_length": 494.6380310058594, |
| "completions/mean_terminated_length": 486.5718078613281, |
| "completions/min_length": 231.0, |
| "completions/min_terminated_length": 231.0, |
| "entropy": 0.11871337890625, |
| "epoch": 0.22811519817507842, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0348508358001709, |
| "kl": 0.0009682774543762207, |
| "learning_rate": 1.7741416106390828e-05, |
| "loss": -0.0021, |
| "num_tokens": 10537447.0, |
| "reward": 0.9557062387466431, |
| "reward_std": 0.22823719680309296, |
| "rewards/cross_solution_unittest_reward/mean": 0.4154369905591011, |
| "rewards/cross_solution_unittest_reward/std": 0.20701014250516891, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 25 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1757.0, |
| "completions/max_terminated_length": 1757.0, |
| "completions/mean_length": 497.625, |
| "completions/mean_terminated_length": 497.625, |
| "completions/min_length": 213.0, |
| "completions/min_terminated_length": 213.0, |
| "entropy": 0.11309814453125, |
| "epoch": 0.23723980610208156, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028710953891277313, |
| "kl": 0.0007503032684326172, |
| "learning_rate": 1.7557495743542586e-05, |
| "loss": 0.0115, |
| "num_tokens": 10895703.0, |
| "reward": 0.8731624484062195, |
| "reward_std": 0.24689321219921112, |
| "rewards/cross_solution_unittest_reward/mean": 0.3826525807380676, |
| "rewards/cross_solution_unittest_reward/std": 0.24787958711385727, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 902.0, |
| "completions/max_terminated_length": 902.0, |
| "completions/mean_length": 473.3671875, |
| "completions/mean_terminated_length": 473.3671875, |
| "completions/min_length": 276.0, |
| "completions/min_terminated_length": 276.0, |
| "entropy": 0.111358642578125, |
| "epoch": 0.24636441402908468, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030619844794273376, |
| "kl": 0.0008200407028198242, |
| "learning_rate": 1.736741137876405e-05, |
| "loss": 0.002, |
| "num_tokens": 11274364.0, |
| "reward": 1.0031293630599976, |
| "reward_std": 0.2526843547821045, |
| "rewards/cross_solution_unittest_reward/mean": 0.44217514991760254, |
| "rewards/cross_solution_unittest_reward/std": 0.21224447339773178, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 27 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 957.0, |
| "completions/max_terminated_length": 957.0, |
| "completions/mean_length": 506.1458435058594, |
| "completions/mean_terminated_length": 506.1458435058594, |
| "completions/min_length": 219.0, |
| "completions/min_terminated_length": 219.0, |
| "entropy": 0.1076812744140625, |
| "epoch": 0.2554890219560878, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.026609111577272415, |
| "kl": 0.0008921623229980469, |
| "learning_rate": 1.7171318047589637e-05, |
| "loss": -0.0014, |
| "num_tokens": 11706916.0, |
| "reward": 1.137014389038086, |
| "reward_std": 0.2074248045682907, |
| "rewards/cross_solution_unittest_reward/mean": 0.4929664433002472, |
| "rewards/cross_solution_unittest_reward/std": 0.26993829011917114, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 28 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 952.0, |
| "completions/max_terminated_length": 952.0, |
| "completions/mean_length": 489.57293701171875, |
| "completions/mean_terminated_length": 489.57293701171875, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 256.0, |
| "entropy": 0.117828369140625, |
| "epoch": 0.26461362988309095, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031492725014686584, |
| "kl": 0.001159071922302246, |
| "learning_rate": 1.696937568655294e-05, |
| "loss": -0.0028, |
| "num_tokens": 12102904.0, |
| "reward": 1.0333976745605469, |
| "reward_std": 0.2310364693403244, |
| "rewards/cross_solution_unittest_reward/mean": 0.4447094053030014, |
| "rewards/cross_solution_unittest_reward/std": 0.29005488753318787, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 29 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1168.0, |
| "completions/mean_length": 544.9635620117188, |
| "completions/mean_terminated_length": 529.0523681640625, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.103607177734375, |
| "epoch": 0.2737382378100941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028927048668265343, |
| "kl": 0.0009303092956542969, |
| "learning_rate": 1.6761749002740195e-05, |
| "loss": 0.0036, |
| "num_tokens": 12531050.0, |
| "reward": 0.8030904531478882, |
| "reward_std": 0.2894265055656433, |
| "rewards/cross_solution_unittest_reward/mean": 0.35538169741630554, |
| "rewards/cross_solution_unittest_reward/std": 0.27038178592920303, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 946.0, |
| "completions/mean_length": 486.05731201171875, |
| "completions/mean_terminated_length": 469.83770751953125, |
| "completions/min_length": 236.0, |
| "completions/min_terminated_length": 236.0, |
| "entropy": 0.1143035888671875, |
| "epoch": 0.28286284573709725, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030626529827713966, |
| "kl": 0.0011305809020996094, |
| "learning_rate": 1.6548607339452853e-05, |
| "loss": 0.0161, |
| "num_tokens": 12901128.0, |
| "reward": 1.0075210332870483, |
| "reward_std": 0.2209041714668274, |
| "rewards/cross_solution_unittest_reward/mean": 0.4487333372235298, |
| "rewards/cross_solution_unittest_reward/std": 0.21068686991930008, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 31 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 954.0, |
| "completions/max_terminated_length": 954.0, |
| "completions/mean_length": 481.5677185058594, |
| "completions/mean_terminated_length": 481.5677185058594, |
| "completions/min_length": 237.0, |
| "completions/min_terminated_length": 237.0, |
| "entropy": 0.103546142578125, |
| "epoch": 0.2919874536641004, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03601611778140068, |
| "kl": 0.0013216733932495117, |
| "learning_rate": 1.6330124538088705e-05, |
| "loss": 0.0017, |
| "num_tokens": 13354434.0, |
| "reward": 1.051571011543274, |
| "reward_std": 0.3614152669906616, |
| "rewards/cross_solution_unittest_reward/mean": 0.4522690027952194, |
| "rewards/cross_solution_unittest_reward/std": 0.24180202186107635, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 773.0, |
| "completions/max_terminated_length": 773.0, |
| "completions/mean_length": 469.72137451171875, |
| "completions/mean_terminated_length": 469.72137451171875, |
| "completions/min_length": 241.0, |
| "completions/min_terminated_length": 241.0, |
| "entropy": 0.115203857421875, |
| "epoch": 0.3011120615911035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02829568460583687, |
| "kl": 0.0014170408248901367, |
| "learning_rate": 1.6106478796354382e-05, |
| "loss": 0.0034, |
| "num_tokens": 13732783.0, |
| "reward": 0.921411395072937, |
| "reward_std": 0.17318974435329437, |
| "rewards/cross_solution_unittest_reward/mean": 0.4217027798295021, |
| "rewards/cross_solution_unittest_reward/std": 0.22807485610246658, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 33 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 839.0, |
| "completions/max_terminated_length": 839.0, |
| "completions/mean_length": 488.7578125, |
| "completions/mean_terminated_length": 488.7578125, |
| "completions/min_length": 217.0, |
| "completions/min_terminated_length": 217.0, |
| "entropy": 0.1075286865234375, |
| "epoch": 0.31023666951810663, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029855074360966682, |
| "kl": 0.001324295997619629, |
| "learning_rate": 1.5877852522924733e-05, |
| "loss": -0.0055, |
| "num_tokens": 14159730.0, |
| "reward": 1.081244707107544, |
| "reward_std": 0.18814358115196228, |
| "rewards/cross_solution_unittest_reward/mean": 0.4768352806568146, |
| "rewards/cross_solution_unittest_reward/std": 0.22413263469934464, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 34 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1815.0, |
| "completions/max_terminated_length": 1815.0, |
| "completions/mean_length": 510.74481201171875, |
| "completions/mean_terminated_length": 510.74481201171875, |
| "completions/min_length": 280.0, |
| "completions/min_terminated_length": 280.0, |
| "entropy": 0.1134490966796875, |
| "epoch": 0.3193612774451098, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03141755983233452, |
| "kl": 0.0015311241149902344, |
| "learning_rate": 1.5644432188667695e-05, |
| "loss": 0.0011, |
| "num_tokens": 14547176.0, |
| "reward": 0.9383748769760132, |
| "reward_std": 0.2669951617717743, |
| "rewards/cross_solution_unittest_reward/mean": 0.3990442007780075, |
| "rewards/cross_solution_unittest_reward/std": 0.24042869359254837, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 35 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1357.0, |
| "completions/mean_length": 537.3489990234375, |
| "completions/mean_terminated_length": 529.394287109375, |
| "completions/min_length": 273.0, |
| "completions/min_terminated_length": 273.0, |
| "entropy": 0.1122283935546875, |
| "epoch": 0.32848588537211293, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030957432463765144, |
| "kl": 0.0015711784362792969, |
| "learning_rate": 1.5406408174555978e-05, |
| "loss": -0.0053, |
| "num_tokens": 14975526.0, |
| "reward": 0.9109244346618652, |
| "reward_std": 0.25726160407066345, |
| "rewards/cross_solution_unittest_reward/mean": 0.3915872722864151, |
| "rewards/cross_solution_unittest_reward/std": 0.22853311151266098, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 36 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 2463.0, |
| "completions/mean_length": 520.1276245117188, |
| "completions/mean_terminated_length": 512.1279296875, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.097320556640625, |
| "epoch": 0.3376104932991161, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032772429287433624, |
| "kl": 0.0015528202056884766, |
| "learning_rate": 1.5163974616389621e-05, |
| "loss": 0.0123, |
| "num_tokens": 15386143.0, |
| "reward": 0.9813830256462097, |
| "reward_std": 0.22931739687919617, |
| "rewards/cross_solution_unittest_reward/mean": 0.43425410240888596, |
| "rewards/cross_solution_unittest_reward/std": 0.22800295799970627, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 37 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 887.0, |
| "completions/max_terminated_length": 887.0, |
| "completions/mean_length": 485.6380310058594, |
| "completions/mean_terminated_length": 485.6380310058594, |
| "completions/min_length": 205.0, |
| "completions/min_terminated_length": 205.0, |
| "entropy": 0.1133270263671875, |
| "epoch": 0.34673510122611917, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03404747322201729, |
| "kl": 0.0015780925750732422, |
| "learning_rate": 1.491732924645604e-05, |
| "loss": 0.0191, |
| "num_tokens": 15770812.0, |
| "reward": 0.9875783920288086, |
| "reward_std": 0.2382659763097763, |
| "rewards/cross_solution_unittest_reward/mean": 0.4320690929889679, |
| "rewards/cross_solution_unittest_reward/std": 0.2256365790963173, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 38 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1240.0, |
| "completions/max_terminated_length": 1240.0, |
| "completions/mean_length": 473.8984375, |
| "completions/mean_terminated_length": 473.8984375, |
| "completions/min_length": 179.0, |
| "completions/min_terminated_length": 179.0, |
| "entropy": 0.116729736328125, |
| "epoch": 0.3558597091531223, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03682932257652283, |
| "kl": 0.003524303436279297, |
| "learning_rate": 1.4666673232256738e-05, |
| "loss": -0.0084, |
| "num_tokens": 16082925.0, |
| "reward": 0.8647634387016296, |
| "reward_std": 0.3210960626602173, |
| "rewards/cross_solution_unittest_reward/mean": 0.37845826894044876, |
| "rewards/cross_solution_unittest_reward/std": 0.2232651561498642, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 39 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2045.0, |
| "completions/max_terminated_length": 2045.0, |
| "completions/mean_length": 503.125, |
| "completions/mean_terminated_length": 503.125, |
| "completions/min_length": 202.0, |
| "completions/min_terminated_length": 202.0, |
| "entropy": 0.102325439453125, |
| "epoch": 0.36498431708012546, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04535383731126785, |
| "kl": 0.003467082977294922, |
| "learning_rate": 1.4412211012432213e-05, |
| "loss": 0.0068, |
| "num_tokens": 16496789.0, |
| "reward": 0.8126609325408936, |
| "reward_std": 0.26924118399620056, |
| "rewards/cross_solution_unittest_reward/mean": 0.3569253906607628, |
| "rewards/cross_solution_unittest_reward/std": 0.2864082232117653, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 40 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 958.0, |
| "completions/max_terminated_length": 958.0, |
| "completions/mean_length": 511.31512451171875, |
| "completions/mean_terminated_length": 511.31512451171875, |
| "completions/min_length": 248.0, |
| "completions/min_terminated_length": 248.0, |
| "entropy": 0.1136627197265625, |
| "epoch": 0.3741089250071286, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033672649413347244, |
| "kl": 0.0028324127197265625, |
| "learning_rate": 1.4154150130018867e-05, |
| "loss": -0.0017, |
| "num_tokens": 16912990.0, |
| "reward": 0.7562817335128784, |
| "reward_std": 0.20014888048171997, |
| "rewards/cross_solution_unittest_reward/mean": 0.3420030549168587, |
| "rewards/cross_solution_unittest_reward/std": 0.21658644825220108, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 41 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1215.0, |
| "completions/max_terminated_length": 1215.0, |
| "completions/mean_length": 506.51043701171875, |
| "completions/mean_terminated_length": 506.51043701171875, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "entropy": 0.093170166015625, |
| "epoch": 0.38323353293413176, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03319159895181656, |
| "kl": 0.0018248558044433594, |
| "learning_rate": 1.3892701063173917e-05, |
| "loss": -0.0041, |
| "num_tokens": 17357914.0, |
| "reward": 0.9257922172546387, |
| "reward_std": 0.2104812115430832, |
| "rewards/cross_solution_unittest_reward/mean": 0.4091387465596199, |
| "rewards/cross_solution_unittest_reward/std": 0.23963715881109238, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 42 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 799.0, |
| "completions/max_terminated_length": 799.0, |
| "completions/mean_length": 478.70574951171875, |
| "completions/mean_terminated_length": 478.70574951171875, |
| "completions/min_length": 249.0, |
| "completions/min_terminated_length": 249.0, |
| "entropy": 0.105377197265625, |
| "epoch": 0.39235814086113485, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03309491649270058, |
| "kl": 0.00234222412109375, |
| "learning_rate": 1.362807705350641e-05, |
| "loss": -0.0027, |
| "num_tokens": 17696089.0, |
| "reward": 1.0697866678237915, |
| "reward_std": 0.21351736783981323, |
| "rewards/cross_solution_unittest_reward/mean": 0.46968914568424225, |
| "rewards/cross_solution_unittest_reward/std": 0.25238344073295593, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 43 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 2798.0, |
| "completions/mean_length": 519.0208740234375, |
| "completions/mean_terminated_length": 511.0182800292969, |
| "completions/min_length": 224.0, |
| "completions/min_terminated_length": 224.0, |
| "entropy": 0.095550537109375, |
| "epoch": 0.401482748788138, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029689131304621696, |
| "kl": 0.0020627975463867188, |
| "learning_rate": 1.3360493932154301e-05, |
| "loss": -0.0021, |
| "num_tokens": 18106297.0, |
| "reward": 0.7189862728118896, |
| "reward_std": 0.21532303094863892, |
| "rewards/cross_solution_unittest_reward/mean": 0.3341630734503269, |
| "rewards/cross_solution_unittest_reward/std": 0.210732139647007, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 44 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1472.0, |
| "completions/max_terminated_length": 1472.0, |
| "completions/mean_length": 467.91668701171875, |
| "completions/mean_terminated_length": 467.91668701171875, |
| "completions/min_length": 196.0, |
| "completions/min_terminated_length": 196.0, |
| "entropy": 0.1134033203125, |
| "epoch": 0.41060735671514115, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033839646726846695, |
| "kl": 0.0030853748321533203, |
| "learning_rate": 1.3090169943749475e-05, |
| "loss": 0.021, |
| "num_tokens": 18488529.0, |
| "reward": 1.0240888595581055, |
| "reward_std": 0.21602611243724823, |
| "rewards/cross_solution_unittest_reward/mean": 0.4490320533514023, |
| "rewards/cross_solution_unittest_reward/std": 0.2549639642238617, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 45 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1161.0, |
| "completions/mean_length": 520.1119995117188, |
| "completions/mean_terminated_length": 512.1123046875, |
| "completions/min_length": 281.0, |
| "completions/min_terminated_length": 281.0, |
| "entropy": 0.0985565185546875, |
| "epoch": 0.4197319646421443, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031324755400419235, |
| "kl": 0.0023124217987060547, |
| "learning_rate": 1.2817325568414299e-05, |
| "loss": -0.0063, |
| "num_tokens": 18908484.0, |
| "reward": 0.8984124064445496, |
| "reward_std": 0.31652843952178955, |
| "rewards/cross_solution_unittest_reward/mean": 0.394581638276577, |
| "rewards/cross_solution_unittest_reward/std": 0.2338028848171234, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 46 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.921875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1145.0, |
| "completions/mean_length": 528.3125, |
| "completions/mean_terminated_length": 488.0000305175781, |
| "completions/min_length": 215.0, |
| "completions/min_terminated_length": 215.0, |
| "entropy": 0.11846923828125, |
| "epoch": 0.42885657256914744, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03654487058520317, |
| "kl": 0.003631591796875, |
| "learning_rate": 1.2542183341934873e-05, |
| "loss": 0.0342, |
| "num_tokens": 19336532.0, |
| "reward": 0.7277412414550781, |
| "reward_std": 0.2056652158498764, |
| "rewards/cross_solution_unittest_reward/mean": 0.3252406492829323, |
| "rewards/cross_solution_unittest_reward/std": 0.24651427567005157, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 47 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1054.0, |
| "completions/max_terminated_length": 1054.0, |
| "completions/mean_length": 492.11199951171875, |
| "completions/mean_terminated_length": 492.11199951171875, |
| "completions/min_length": 262.0, |
| "completions/min_terminated_length": 262.0, |
| "entropy": 0.1041717529296875, |
| "epoch": 0.43798118049615054, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03339504823088646, |
| "kl": 0.0033910274505615234, |
| "learning_rate": 1.2264967674257647e-05, |
| "loss": 0.0055, |
| "num_tokens": 19727471.0, |
| "reward": 0.9818544387817383, |
| "reward_std": 0.26466554403305054, |
| "rewards/cross_solution_unittest_reward/mean": 0.4283819645643234, |
| "rewards/cross_solution_unittest_reward/std": 0.22476153075695038, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1095.0, |
| "completions/mean_length": 502.90625, |
| "completions/mean_terminated_length": 494.86163330078125, |
| "completions/min_length": 262.0, |
| "completions/min_terminated_length": 262.0, |
| "entropy": 0.1012725830078125, |
| "epoch": 0.4471057884231537, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 18.39661979675293, |
| "kl": 0.8277513980865479, |
| "learning_rate": 1.1985904666457455e-05, |
| "loss": 0.0468, |
| "num_tokens": 20142435.0, |
| "reward": 0.7730751037597656, |
| "reward_std": 0.30047422647476196, |
| "rewards/cross_solution_unittest_reward/mean": 0.33403923362493515, |
| "rewards/cross_solution_unittest_reward/std": 0.25003235042095184, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 49 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1312.0, |
| "completions/max_terminated_length": 1312.0, |
| "completions/mean_length": 509.84375, |
| "completions/mean_terminated_length": 509.84375, |
| "completions/min_length": 248.0, |
| "completions/min_terminated_length": 248.0, |
| "entropy": 0.09967041015625, |
| "epoch": 0.45623039635015683, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03424382582306862, |
| "kl": 0.0036292076110839844, |
| "learning_rate": 1.170522192632624e-05, |
| "loss": -0.0109, |
| "num_tokens": 20554295.0, |
| "reward": 0.9384050369262695, |
| "reward_std": 0.2790818512439728, |
| "rewards/cross_solution_unittest_reward/mean": 0.41122280806303024, |
| "rewards/cross_solution_unittest_reward/std": 0.28970006108283997, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 909.0, |
| "completions/max_terminated_length": 909.0, |
| "completions/mean_length": 509.796875, |
| "completions/mean_terminated_length": 509.796875, |
| "completions/min_length": 173.0, |
| "completions/min_terminated_length": 173.0, |
| "entropy": 0.0945892333984375, |
| "epoch": 0.46535500427716, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030270231887698174, |
| "kl": 0.003202676773071289, |
| "learning_rate": 1.1423148382732854e-05, |
| "loss": 0.0027, |
| "num_tokens": 20974745.0, |
| "reward": 1.0798063278198242, |
| "reward_std": 0.2213771790266037, |
| "rewards/cross_solution_unittest_reward/mean": 0.4737387150526047, |
| "rewards/cross_solution_unittest_reward/std": 0.18860329687595367, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1354.0, |
| "completions/mean_length": 513.109375, |
| "completions/mean_terminated_length": 505.0914001464844, |
| "completions/min_length": 251.0, |
| "completions/min_terminated_length": 251.0, |
| "entropy": 0.099945068359375, |
| "epoch": 0.4744796122041631, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03432578966021538, |
| "kl": 0.0032448768615722656, |
| "learning_rate": 1.1139914098905406e-05, |
| "loss": 0.0178, |
| "num_tokens": 21378419.0, |
| "reward": 1.0716464519500732, |
| "reward_std": 0.2842645049095154, |
| "rewards/cross_solution_unittest_reward/mean": 0.47282470762729645, |
| "rewards/cross_solution_unittest_reward/std": 0.23804496228694916, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 52 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1391.0, |
| "completions/max_terminated_length": 1391.0, |
| "completions/mean_length": 487.3828125, |
| "completions/mean_terminated_length": 487.3828125, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.0956878662109375, |
| "epoch": 0.4836042201311662, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03196337819099426, |
| "kl": 0.0026366710662841797, |
| "learning_rate": 1.08557500847884e-05, |
| "loss": -0.0004, |
| "num_tokens": 21779142.0, |
| "reward": 0.8248944282531738, |
| "reward_std": 0.2299119085073471, |
| "rewards/cross_solution_unittest_reward/mean": 0.3687998875975609, |
| "rewards/cross_solution_unittest_reward/std": 0.22956574708223343, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 53 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1238.0, |
| "completions/max_terminated_length": 1238.0, |
| "completions/mean_length": 536.3932495117188, |
| "completions/mean_terminated_length": 536.3932495117188, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.1009521484375, |
| "epoch": 0.49272882805816937, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03287142142653465, |
| "kl": 0.003429412841796875, |
| "learning_rate": 1.0570888108627682e-05, |
| "loss": -0.0033, |
| "num_tokens": 22241477.0, |
| "reward": 0.8158465623855591, |
| "reward_std": 0.2910717725753784, |
| "rewards/cross_solution_unittest_reward/mean": 0.358261875808239, |
| "rewards/cross_solution_unittest_reward/std": 0.21051427721977234, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 54 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 898.0, |
| "completions/max_terminated_length": 898.0, |
| "completions/mean_length": 491.3802185058594, |
| "completions/mean_terminated_length": 491.3802185058594, |
| "completions/min_length": 219.0, |
| "completions/min_terminated_length": 219.0, |
| "entropy": 0.1052398681640625, |
| "epoch": 0.5018534359851725, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03173370659351349, |
| "kl": 0.0028574466705322266, |
| "learning_rate": 1.0285560507936962e-05, |
| "loss": -0.0091, |
| "num_tokens": 22619007.0, |
| "reward": 0.8488973379135132, |
| "reward_std": 0.24665257334709167, |
| "rewards/cross_solution_unittest_reward/mean": 0.37161923199892044, |
| "rewards/cross_solution_unittest_reward/std": 0.21222124248743057, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 55 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 932.0, |
| "completions/max_terminated_length": 932.0, |
| "completions/mean_length": 494.5208435058594, |
| "completions/mean_terminated_length": 494.5208435058594, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "entropy": 0.1017913818359375, |
| "epoch": 0.5109780439121756, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.045224472880363464, |
| "kl": 0.00625157356262207, |
| "learning_rate": 1e-05, |
| "loss": 0.0093, |
| "num_tokens": 22993871.0, |
| "reward": 0.9006222486495972, |
| "reward_std": 0.21599134802818298, |
| "rewards/cross_solution_unittest_reward/mean": 0.40076694637537, |
| "rewards/cross_solution_unittest_reward/std": 0.22537627071142197, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 56 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 981.0, |
| "completions/mean_length": 510.2161560058594, |
| "completions/mean_terminated_length": 502.19061279296875, |
| "completions/min_length": 308.0, |
| "completions/min_terminated_length": 308.0, |
| "entropy": 0.1026153564453125, |
| "epoch": 0.5201026518391788, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033613454550504684, |
| "kl": 0.0032994747161865234, |
| "learning_rate": 9.71443949206304e-06, |
| "loss": -0.0064, |
| "num_tokens": 23442418.0, |
| "reward": 0.9552373290061951, |
| "reward_std": 0.21273988485336304, |
| "rewards/cross_solution_unittest_reward/mean": 0.4271874502301216, |
| "rewards/cross_solution_unittest_reward/std": 0.20565658062696457, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 925.0, |
| "completions/mean_length": 494.5208435058594, |
| "completions/mean_terminated_length": 478.3455505371094, |
| "completions/min_length": 258.0, |
| "completions/min_terminated_length": 258.0, |
| "entropy": 0.117401123046875, |
| "epoch": 0.5292272597661819, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03614190220832825, |
| "kl": 0.0036954879760742188, |
| "learning_rate": 9.42911189137232e-06, |
| "loss": 0.0155, |
| "num_tokens": 23857530.0, |
| "reward": 0.9477154016494751, |
| "reward_std": 0.20985114574432373, |
| "rewards/cross_solution_unittest_reward/mean": 0.41357675194740295, |
| "rewards/cross_solution_unittest_reward/std": 0.2724653407931328, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 899.0, |
| "completions/mean_length": 474.82293701171875, |
| "completions/mean_terminated_length": 466.7049560546875, |
| "completions/min_length": 242.0, |
| "completions/min_terminated_length": 242.0, |
| "entropy": 0.1109161376953125, |
| "epoch": 0.5383518676931851, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03441567346453667, |
| "kl": 0.00311279296875, |
| "learning_rate": 9.144249915211605e-06, |
| "loss": 0.0062, |
| "num_tokens": 24261294.0, |
| "reward": 0.9669393301010132, |
| "reward_std": 0.22922192513942719, |
| "rewards/cross_solution_unittest_reward/mean": 0.4128674417734146, |
| "rewards/cross_solution_unittest_reward/std": 0.2560960501432419, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 59 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 854.0, |
| "completions/max_terminated_length": 854.0, |
| "completions/mean_length": 497.9609375, |
| "completions/mean_terminated_length": 497.9609375, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.0975494384765625, |
| "epoch": 0.5474764756201882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033020906150341034, |
| "kl": 0.002737283706665039, |
| "learning_rate": 8.860085901094595e-06, |
| "loss": -0.0018, |
| "num_tokens": 24696543.0, |
| "reward": 0.9655224680900574, |
| "reward_std": 0.28828054666519165, |
| "rewards/cross_solution_unittest_reward/mean": 0.4253334328532219, |
| "rewards/cross_solution_unittest_reward/std": 0.23384006321430206, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 999.0, |
| "completions/max_terminated_length": 999.0, |
| "completions/mean_length": 524.890625, |
| "completions/mean_terminated_length": 524.890625, |
| "completions/min_length": 263.0, |
| "completions/min_terminated_length": 263.0, |
| "entropy": 0.096435546875, |
| "epoch": 0.5566010835471913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03234833851456642, |
| "kl": 0.003005504608154297, |
| "learning_rate": 8.576851617267151e-06, |
| "loss": 0.0086, |
| "num_tokens": 25143405.0, |
| "reward": 1.010263442993164, |
| "reward_std": 0.23946420848369598, |
| "rewards/cross_solution_unittest_reward/mean": 0.44504720717668533, |
| "rewards/cross_solution_unittest_reward/std": 0.2548332065343857, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 61 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1025.0, |
| "completions/mean_length": 530.8958740234375, |
| "completions/mean_terminated_length": 522.92431640625, |
| "completions/min_length": 232.0, |
| "completions/min_terminated_length": 232.0, |
| "entropy": 0.10467529296875, |
| "epoch": 0.5657256914741945, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03223549574613571, |
| "kl": 0.002941131591796875, |
| "learning_rate": 8.294778073673762e-06, |
| "loss": 0.007, |
| "num_tokens": 25572309.0, |
| "reward": 0.7772404551506042, |
| "reward_std": 0.2317599058151245, |
| "rewards/cross_solution_unittest_reward/mean": 0.35900117829442024, |
| "rewards/cross_solution_unittest_reward/std": 0.1959025040268898, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1132.0, |
| "completions/mean_length": 478.97137451171875, |
| "completions/mean_terminated_length": 470.8642272949219, |
| "completions/min_length": 156.0, |
| "completions/min_terminated_length": 156.0, |
| "entropy": 0.1028900146484375, |
| "epoch": 0.5748502994011976, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034286729991436005, |
| "kl": 0.0025823116302490234, |
| "learning_rate": 8.014095333542548e-06, |
| "loss": 0.0114, |
| "num_tokens": 25945338.0, |
| "reward": 0.8456071615219116, |
| "reward_std": 0.2458418607711792, |
| "rewards/cross_solution_unittest_reward/mean": 0.37883561104536057, |
| "rewards/cross_solution_unittest_reward/std": 0.22371377050876617, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 63 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 855.0, |
| "completions/max_terminated_length": 855.0, |
| "completions/mean_length": 491.22137451171875, |
| "completions/mean_terminated_length": 491.22137451171875, |
| "completions/min_length": 309.0, |
| "completions/min_terminated_length": 309.0, |
| "entropy": 0.0994110107421875, |
| "epoch": 0.5839749073282008, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030411940068006516, |
| "kl": 0.0029740333557128906, |
| "learning_rate": 7.735032325742355e-06, |
| "loss": 0.0071, |
| "num_tokens": 26337343.0, |
| "reward": 1.2452019453048706, |
| "reward_std": 0.28348508477211, |
| "rewards/cross_solution_unittest_reward/mean": 0.5320437103509903, |
| "rewards/cross_solution_unittest_reward/std": 0.20597976446151733, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 64 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1100.0, |
| "completions/max_terminated_length": 1100.0, |
| "completions/mean_length": 481.04949951171875, |
| "completions/mean_terminated_length": 481.04949951171875, |
| "completions/min_length": 232.0, |
| "completions/min_terminated_length": 232.0, |
| "entropy": 0.1112823486328125, |
| "epoch": 0.5930995152552039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03709479793906212, |
| "kl": 0.0034236907958984375, |
| "learning_rate": 7.4578166580651335e-06, |
| "loss": 0.0018, |
| "num_tokens": 26727642.0, |
| "reward": 1.0363634824752808, |
| "reward_std": 0.2708354592323303, |
| "rewards/cross_solution_unittest_reward/mean": 0.4516971707344055, |
| "rewards/cross_solution_unittest_reward/std": 0.30414242297410965, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 65 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1223.0, |
| "completions/mean_length": 540.4427490234375, |
| "completions/mean_terminated_length": 516.4776611328125, |
| "completions/min_length": 225.0, |
| "completions/min_terminated_length": 225.0, |
| "entropy": 0.102630615234375, |
| "epoch": 0.602224123182207, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031675394624471664, |
| "kl": 0.0034074783325195312, |
| "learning_rate": 7.182674431585703e-06, |
| "loss": -0.0053, |
| "num_tokens": 27207164.0, |
| "reward": 1.1464284658432007, |
| "reward_std": 0.23834940791130066, |
| "rewards/cross_solution_unittest_reward/mean": 0.4887780696153641, |
| "rewards/cross_solution_unittest_reward/std": 0.26997726410627365, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1062.0, |
| "completions/max_terminated_length": 1062.0, |
| "completions/mean_length": 492.33856201171875, |
| "completions/mean_terminated_length": 492.33856201171875, |
| "completions/min_length": 217.0, |
| "completions/min_terminated_length": 217.0, |
| "entropy": 0.098785400390625, |
| "epoch": 0.6113487311092102, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03432027995586395, |
| "kl": 0.004305839538574219, |
| "learning_rate": 6.909830056250527e-06, |
| "loss": 0.0091, |
| "num_tokens": 27661350.0, |
| "reward": 1.2568825483322144, |
| "reward_std": 0.3327004909515381, |
| "rewards/cross_solution_unittest_reward/mean": 0.5392695367336273, |
| "rewards/cross_solution_unittest_reward/std": 0.24271728843450546, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 953.0, |
| "completions/max_terminated_length": 953.0, |
| "completions/mean_length": 515.4479370117188, |
| "completions/mean_terminated_length": 515.4479370117188, |
| "completions/min_length": 270.0, |
| "completions/min_terminated_length": 270.0, |
| "entropy": 0.0980682373046875, |
| "epoch": 0.6204733390362133, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03175772726535797, |
| "kl": 0.0025773048400878906, |
| "learning_rate": 6.639506067845698e-06, |
| "loss": 0.0052, |
| "num_tokens": 28061562.0, |
| "reward": 1.0176376104354858, |
| "reward_std": 0.2250736653804779, |
| "rewards/cross_solution_unittest_reward/mean": 0.44822562485933304, |
| "rewards/cross_solution_unittest_reward/std": 0.2024165391921997, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 937.0, |
| "completions/max_terminated_length": 937.0, |
| "completions/mean_length": 493.3567810058594, |
| "completions/mean_terminated_length": 493.3567810058594, |
| "completions/min_length": 291.0, |
| "completions/min_terminated_length": 291.0, |
| "entropy": 0.0994415283203125, |
| "epoch": 0.6295979469632165, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033092062920331955, |
| "kl": 0.0027437210083007812, |
| "learning_rate": 6.3719229464935915e-06, |
| "loss": 0.0023, |
| "num_tokens": 28497043.0, |
| "reward": 1.0380425453186035, |
| "reward_std": 0.23625622689723969, |
| "rewards/cross_solution_unittest_reward/mean": 0.4579319655895233, |
| "rewards/cross_solution_unittest_reward/std": 0.20026205480098724, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 69 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1090.0, |
| "completions/max_terminated_length": 1090.0, |
| "completions/mean_length": 518.9921875, |
| "completions/mean_terminated_length": 518.9921875, |
| "completions/min_length": 243.0, |
| "completions/min_terminated_length": 243.0, |
| "entropy": 0.0950927734375, |
| "epoch": 0.6387225548902196, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03034958802163601, |
| "kl": 0.0027964115142822266, |
| "learning_rate": 6.107298936826086e-06, |
| "loss": -0.001, |
| "num_tokens": 28942272.0, |
| "reward": 1.1260254383087158, |
| "reward_std": 0.3392567038536072, |
| "rewards/cross_solution_unittest_reward/mean": 0.4806702435016632, |
| "rewards/cross_solution_unittest_reward/std": 0.2181769162416458, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 70 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 983.0, |
| "completions/mean_length": 520.0963745117188, |
| "completions/mean_terminated_length": 504.05499267578125, |
| "completions/min_length": 282.0, |
| "completions/min_terminated_length": 282.0, |
| "entropy": 0.1016082763671875, |
| "epoch": 0.6478471628172227, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03144148737192154, |
| "kl": 0.0028443336486816406, |
| "learning_rate": 5.845849869981137e-06, |
| "loss": 0.0139, |
| "num_tokens": 29354421.0, |
| "reward": 0.683957576751709, |
| "reward_std": 0.19793997704982758, |
| "rewards/cross_solution_unittest_reward/mean": 0.31807853654026985, |
| "rewards/cross_solution_unittest_reward/std": 0.18481089919805527, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1151.0, |
| "completions/max_terminated_length": 1151.0, |
| "completions/mean_length": 519.0833740234375, |
| "completions/mean_terminated_length": 519.0833740234375, |
| "completions/min_length": 273.0, |
| "completions/min_terminated_length": 273.0, |
| "entropy": 0.10321044921875, |
| "epoch": 0.6569717707442259, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.42725464701652527, |
| "kl": 0.03642606735229492, |
| "learning_rate": 5.587788987567785e-06, |
| "loss": 0.0002, |
| "num_tokens": 29764661.0, |
| "reward": 1.0907951593399048, |
| "reward_std": 0.25230711698532104, |
| "rewards/cross_solution_unittest_reward/mean": 0.47279611229896545, |
| "rewards/cross_solution_unittest_reward/std": 0.2695095092058182, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 72 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 791.0, |
| "completions/max_terminated_length": 791.0, |
| "completions/mean_length": 481.8125, |
| "completions/mean_terminated_length": 481.8125, |
| "completions/min_length": 261.0, |
| "completions/min_terminated_length": 261.0, |
| "entropy": 0.1062469482421875, |
| "epoch": 0.666096378671229, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033676039427518845, |
| "kl": 0.003365039825439453, |
| "learning_rate": 5.333326767743263e-06, |
| "loss": 0.0049, |
| "num_tokens": 30167253.0, |
| "reward": 1.1704354286193848, |
| "reward_std": 0.26924002170562744, |
| "rewards/cross_solution_unittest_reward/mean": 0.503605455160141, |
| "rewards/cross_solution_unittest_reward/std": 0.24150928109884262, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 73 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1028.0, |
| "completions/max_terminated_length": 1028.0, |
| "completions/mean_length": 531.5989990234375, |
| "completions/mean_terminated_length": 531.5989990234375, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.098541259765625, |
| "epoch": 0.6752209865982322, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029249846935272217, |
| "kl": 0.0035860538482666016, |
| "learning_rate": 5.082670753543961e-06, |
| "loss": 0.0039, |
| "num_tokens": 30577827.0, |
| "reward": 1.0187840461730957, |
| "reward_std": 0.27179187536239624, |
| "rewards/cross_solution_unittest_reward/mean": 0.4446810930967331, |
| "rewards/cross_solution_unittest_reward/std": 0.2518850564956665, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1103.0, |
| "completions/max_terminated_length": 1103.0, |
| "completions/mean_length": 513.2943115234375, |
| "completions/mean_terminated_length": 513.2943115234375, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.1102142333984375, |
| "epoch": 0.6843455945252352, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029439395293593407, |
| "kl": 0.002677440643310547, |
| "learning_rate": 4.836025383610382e-06, |
| "loss": 0.0061, |
| "num_tokens": 30956420.0, |
| "reward": 1.0420805215835571, |
| "reward_std": 0.2407897710800171, |
| "rewards/cross_solution_unittest_reward/mean": 0.4538075774908066, |
| "rewards/cross_solution_unittest_reward/std": 0.22261689603328705, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 75 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 958.0, |
| "completions/mean_length": 490.7734375, |
| "completions/mean_terminated_length": 474.57855224609375, |
| "completions/min_length": 233.0, |
| "completions/min_terminated_length": 233.0, |
| "entropy": 0.107452392578125, |
| "epoch": 0.6934702024522383, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032771483063697815, |
| "kl": 0.0031430721282958984, |
| "learning_rate": 4.593591825444028e-06, |
| "loss": 0.0142, |
| "num_tokens": 31310765.0, |
| "reward": 0.8610422015190125, |
| "reward_std": 0.2242804616689682, |
| "rewards/cross_solution_unittest_reward/mean": 0.38095080852508545, |
| "rewards/cross_solution_unittest_reward/std": 0.2320489063858986, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 76 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2310.0, |
| "completions/max_terminated_length": 2310.0, |
| "completions/mean_length": 502.9661560058594, |
| "completions/mean_terminated_length": 502.9661560058594, |
| "completions/min_length": 240.0, |
| "completions/min_terminated_length": 240.0, |
| "entropy": 0.10906982421875, |
| "epoch": 0.7025948103792415, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.036506183445453644, |
| "kl": 0.0029113292694091797, |
| "learning_rate": 4.355567811332311e-06, |
| "loss": -0.0079, |
| "num_tokens": 31718384.0, |
| "reward": 1.0806396007537842, |
| "reward_std": 0.2699912488460541, |
| "rewards/cross_solution_unittest_reward/mean": 0.4672403484582901, |
| "rewards/cross_solution_unittest_reward/std": 0.2118784487247467, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 77 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1254.0, |
| "completions/max_terminated_length": 1254.0, |
| "completions/mean_length": 517.2578125, |
| "completions/mean_terminated_length": 517.2578125, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.09716796875, |
| "epoch": 0.7117194183062446, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034147728234529495, |
| "kl": 0.0033721923828125, |
| "learning_rate": 4.12214747707527e-06, |
| "loss": 0.0056, |
| "num_tokens": 32162219.0, |
| "reward": 1.0363221168518066, |
| "reward_std": 0.3166886568069458, |
| "rewards/cross_solution_unittest_reward/mean": 0.4502222388982773, |
| "rewards/cross_solution_unittest_reward/std": 0.25721245259046555, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 78 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 3357.0, |
| "completions/mean_length": 533.2734375, |
| "completions/mean_terminated_length": 509.251953125, |
| "completions/min_length": 263.0, |
| "completions/min_terminated_length": 263.0, |
| "entropy": 0.1002655029296875, |
| "epoch": 0.7208440262332478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029844792559742928, |
| "kl": 0.0024819374084472656, |
| "learning_rate": 3.893521203645618e-06, |
| "loss": 0.0208, |
| "num_tokens": 32565772.0, |
| "reward": 1.0131210088729858, |
| "reward_std": 0.24623362720012665, |
| "rewards/cross_solution_unittest_reward/mean": 0.44952526688575745, |
| "rewards/cross_solution_unittest_reward/std": 0.233152836561203, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 822.0, |
| "completions/max_terminated_length": 822.0, |
| "completions/mean_length": 475.51824951171875, |
| "completions/mean_terminated_length": 475.51824951171875, |
| "completions/min_length": 223.0, |
| "completions/min_terminated_length": 223.0, |
| "entropy": 0.099822998046875, |
| "epoch": 0.7299686341602509, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03284022584557533, |
| "kl": 0.003690004348754883, |
| "learning_rate": 3.6698754619112974e-06, |
| "loss": -0.0014, |
| "num_tokens": 32926955.0, |
| "reward": 0.9502054452896118, |
| "reward_std": 0.23533646762371063, |
| "rewards/cross_solution_unittest_reward/mean": 0.43129874020814896, |
| "rewards/cross_solution_unittest_reward/std": 0.2330498918890953, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 962.0, |
| "completions/max_terminated_length": 962.0, |
| "completions/mean_length": 487.8177185058594, |
| "completions/mean_terminated_length": 487.8177185058594, |
| "completions/min_length": 200.0, |
| "completions/min_terminated_length": 200.0, |
| "entropy": 0.1024932861328125, |
| "epoch": 0.739093242087254, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033087749034166336, |
| "kl": 0.002620697021484375, |
| "learning_rate": 3.4513926605471504e-06, |
| "loss": -0.0118, |
| "num_tokens": 33311189.0, |
| "reward": 1.0504975318908691, |
| "reward_std": 0.2200670838356018, |
| "rewards/cross_solution_unittest_reward/mean": 0.4615302234888077, |
| "rewards/cross_solution_unittest_reward/std": 0.20018253475427628, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 81 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1020.0, |
| "completions/max_terminated_length": 1020.0, |
| "completions/mean_length": 479.33074951171875, |
| "completions/mean_terminated_length": 479.33074951171875, |
| "completions/min_length": 203.0, |
| "completions/min_terminated_length": 203.0, |
| "entropy": 0.1038665771484375, |
| "epoch": 0.7482178500142572, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03461228311061859, |
| "kl": 0.002687692642211914, |
| "learning_rate": 3.2382509972598087e-06, |
| "loss": 0.0073, |
| "num_tokens": 33701588.0, |
| "reward": 0.9868416786193848, |
| "reward_std": 0.23717579245567322, |
| "rewards/cross_solution_unittest_reward/mean": 0.4351993724703789, |
| "rewards/cross_solution_unittest_reward/std": 0.2459121197462082, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 82 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1788.0, |
| "completions/max_terminated_length": 1788.0, |
| "completions/mean_length": 484.34375, |
| "completions/mean_terminated_length": 484.34375, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.1046905517578125, |
| "epoch": 0.7573424579412603, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032532352954149246, |
| "kl": 0.0031991004943847656, |
| "learning_rate": 3.0306243134470668e-06, |
| "loss": 0.0136, |
| "num_tokens": 34096832.0, |
| "reward": 0.9118785858154297, |
| "reward_std": 0.27310025691986084, |
| "rewards/cross_solution_unittest_reward/mean": 0.39596662670373917, |
| "rewards/cross_solution_unittest_reward/std": 0.23184479027986526, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 83 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 814.0, |
| "completions/mean_length": 497.4140625, |
| "completions/mean_terminated_length": 473.1102294921875, |
| "completions/min_length": 261.0, |
| "completions/min_terminated_length": 261.0, |
| "entropy": 0.112152099609375, |
| "epoch": 0.7664670658682635, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030421627685427666, |
| "kl": 0.0025551319122314453, |
| "learning_rate": 2.8286819524103657e-06, |
| "loss": 0.015, |
| "num_tokens": 34443807.0, |
| "reward": 1.0494276285171509, |
| "reward_std": 0.24891619384288788, |
| "rewards/cross_solution_unittest_reward/mean": 0.4558604210615158, |
| "rewards/cross_solution_unittest_reward/std": 0.21075250953435898, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 84 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 963.0, |
| "completions/mean_length": 498.6770935058594, |
| "completions/mean_terminated_length": 490.6214294433594, |
| "completions/min_length": 267.0, |
| "completions/min_terminated_length": 267.0, |
| "entropy": 0.1007080078125, |
| "epoch": 0.7755916737952666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030757782980799675, |
| "kl": 0.0036520957946777344, |
| "learning_rate": 2.6325886212359496e-06, |
| "loss": 0.0108, |
| "num_tokens": 34874923.0, |
| "reward": 1.0011277198791504, |
| "reward_std": 0.2030499428510666, |
| "rewards/cross_solution_unittest_reward/mean": 0.44105569273233414, |
| "rewards/cross_solution_unittest_reward/std": 0.19068162888288498, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 85 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1026.0, |
| "completions/max_terminated_length": 1026.0, |
| "completions/mean_length": 480.3359375, |
| "completions/mean_terminated_length": 480.3359375, |
| "completions/min_length": 243.0, |
| "completions/min_terminated_length": 243.0, |
| "entropy": 0.108489990234375, |
| "epoch": 0.7847162817222697, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034772977232933044, |
| "kl": 0.0031616687774658203, |
| "learning_rate": 2.4425042564574186e-06, |
| "loss": -0.0031, |
| "num_tokens": 35288116.0, |
| "reward": 0.9796751737594604, |
| "reward_std": 0.23475846648216248, |
| "rewards/cross_solution_unittest_reward/mean": 0.43246912956237793, |
| "rewards/cross_solution_unittest_reward/std": 0.19666722416877747, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 86 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 3303.0, |
| "completions/mean_length": 531.0416870117188, |
| "completions/mean_terminated_length": 523.0704956054688, |
| "completions/min_length": 260.0, |
| "completions/min_terminated_length": 260.0, |
| "entropy": 0.10565185546875, |
| "epoch": 0.7938408896492729, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.039444971829652786, |
| "kl": 0.0033414363861083984, |
| "learning_rate": 2.2585838936091753e-06, |
| "loss": 0.0083, |
| "num_tokens": 35724444.0, |
| "reward": 1.1217626333236694, |
| "reward_std": 0.223866805434227, |
| "rewards/cross_solution_unittest_reward/mean": 0.48780302703380585, |
| "rewards/cross_solution_unittest_reward/std": 0.22542807459831238, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 87 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 878.0, |
| "completions/max_terminated_length": 878.0, |
| "completions/mean_length": 442.8828125, |
| "completions/mean_terminated_length": 442.8828125, |
| "completions/min_length": 235.0, |
| "completions/min_terminated_length": 235.0, |
| "entropy": 0.1145172119140625, |
| "epoch": 0.802965497576276, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03573368489742279, |
| "kl": 0.0030510425567626953, |
| "learning_rate": 2.0809775407772505e-06, |
| "loss": 0.0084, |
| "num_tokens": 36081911.0, |
| "reward": 1.0845204591751099, |
| "reward_std": 0.2791823744773865, |
| "rewards/cross_solution_unittest_reward/mean": 0.461356058716774, |
| "rewards/cross_solution_unittest_reward/std": 0.23068440705537796, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 88 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 852.0, |
| "completions/max_terminated_length": 852.0, |
| "completions/mean_length": 518.3568115234375, |
| "completions/mean_terminated_length": 518.3568115234375, |
| "completions/min_length": 313.0, |
| "completions/min_terminated_length": 313.0, |
| "entropy": 0.093414306640625, |
| "epoch": 0.8120901055032792, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03187531977891922, |
| "kl": 0.002918720245361328, |
| "learning_rate": 1.9098300562505266e-06, |
| "loss": 0.0067, |
| "num_tokens": 36510360.0, |
| "reward": 0.9905633330345154, |
| "reward_std": 0.20145916938781738, |
| "rewards/cross_solution_unittest_reward/mean": 0.4334751144051552, |
| "rewards/cross_solution_unittest_reward/std": 0.23444491624832153, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 89 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 986.0, |
| "completions/max_terminated_length": 986.0, |
| "completions/mean_length": 490.1953125, |
| "completions/mean_terminated_length": 490.1953125, |
| "completions/min_length": 258.0, |
| "completions/min_terminated_length": 258.0, |
| "entropy": 0.0989532470703125, |
| "epoch": 0.8212147134302823, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05914483591914177, |
| "kl": 0.008898019790649414, |
| "learning_rate": 1.74528103037226e-06, |
| "loss": 0.0025, |
| "num_tokens": 36925595.0, |
| "reward": 0.9185196757316589, |
| "reward_std": 0.3170742094516754, |
| "rewards/cross_solution_unittest_reward/mean": 0.3952532112598419, |
| "rewards/cross_solution_unittest_reward/std": 0.2790588438510895, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 90 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 980.0, |
| "completions/max_terminated_length": 980.0, |
| "completions/mean_length": 483.3567810058594, |
| "completions/mean_terminated_length": 483.3567810058594, |
| "completions/min_length": 230.0, |
| "completions/min_terminated_length": 230.0, |
| "entropy": 0.10723876953125, |
| "epoch": 0.8303393213572854, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03479889780282974, |
| "kl": 0.0028510093688964844, |
| "learning_rate": 1.587464671688187e-06, |
| "loss": 0.001, |
| "num_tokens": 37317868.0, |
| "reward": 0.9646342992782593, |
| "reward_std": 0.18958084285259247, |
| "rewards/cross_solution_unittest_reward/mean": 0.4295145273208618, |
| "rewards/cross_solution_unittest_reward/std": 0.2108893096446991, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 91 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 956.0, |
| "completions/max_terminated_length": 956.0, |
| "completions/mean_length": 501.15887451171875, |
| "completions/mean_terminated_length": 501.15887451171875, |
| "completions/min_length": 237.0, |
| "completions/min_terminated_length": 237.0, |
| "entropy": 0.1046600341796875, |
| "epoch": 0.8394639292842886, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030385170131921768, |
| "kl": 0.0032634735107421875, |
| "learning_rate": 1.436509697484111e-06, |
| "loss": 0.0069, |
| "num_tokens": 37734865.0, |
| "reward": 0.9471985697746277, |
| "reward_std": 0.24106745421886444, |
| "rewards/cross_solution_unittest_reward/mean": 0.4128854274749756, |
| "rewards/cross_solution_unittest_reward/std": 0.24737022817134857, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 92 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1438.0, |
| "completions/mean_length": 564.9948120117188, |
| "completions/mean_terminated_length": 549.1884765625, |
| "completions/min_length": 280.0, |
| "completions/min_terminated_length": 280.0, |
| "entropy": 0.087799072265625, |
| "epoch": 0.8485885372112917, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.2799060642719269, |
| "kl": 0.017560720443725586, |
| "learning_rate": 1.2925392288022299e-06, |
| "loss": 0.0064, |
| "num_tokens": 38171079.0, |
| "reward": 0.9303452372550964, |
| "reward_std": 0.21974782645702362, |
| "rewards/cross_solution_unittest_reward/mean": 0.41144028306007385, |
| "rewards/cross_solution_unittest_reward/std": 0.23167048394680023, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 93 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1033.0, |
| "completions/mean_length": 545.6354370117188, |
| "completions/mean_terminated_length": 529.727783203125, |
| "completions/min_length": 204.0, |
| "completions/min_terminated_length": 204.0, |
| "entropy": 0.092437744140625, |
| "epoch": 0.8577131451382949, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02881380170583725, |
| "kl": 0.0027322769165039062, |
| "learning_rate": 1.1556706900218572e-06, |
| "loss": 0.0152, |
| "num_tokens": 38596603.0, |
| "reward": 0.9431265592575073, |
| "reward_std": 0.3126261830329895, |
| "rewards/cross_solution_unittest_reward/mean": 0.4096960350871086, |
| "rewards/cross_solution_unittest_reward/std": 0.274701751768589, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 775.0, |
| "completions/max_terminated_length": 775.0, |
| "completions/mean_length": 492.7734375, |
| "completions/mean_terminated_length": 492.7734375, |
| "completions/min_length": 255.0, |
| "completions/min_terminated_length": 255.0, |
| "entropy": 0.1008758544921875, |
| "epoch": 0.866837753065298, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030817339196801186, |
| "kl": 0.003123760223388672, |
| "learning_rate": 1.0260157130864178e-06, |
| "loss": -0.0065, |
| "num_tokens": 39042508.0, |
| "reward": 0.98272705078125, |
| "reward_std": 0.2458946406841278, |
| "rewards/cross_solution_unittest_reward/mean": 0.42810578644275665, |
| "rewards/cross_solution_unittest_reward/std": 0.19119084626436234, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 95 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1011.0, |
| "completions/max_terminated_length": 1011.0, |
| "completions/mean_length": 520.4036865234375, |
| "completions/mean_terminated_length": 520.4036865234375, |
| "completions/min_length": 241.0, |
| "completions/min_terminated_length": 241.0, |
| "entropy": 0.10015869140625, |
| "epoch": 0.8759623609923011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03871850296854973, |
| "kl": 0.0033326148986816406, |
| "learning_rate": 9.036800464548157e-07, |
| "loss": -0.0045, |
| "num_tokens": 39430183.0, |
| "reward": 0.944609522819519, |
| "reward_std": 0.2243545949459076, |
| "rewards/cross_solution_unittest_reward/mean": 0.40971074998378754, |
| "rewards/cross_solution_unittest_reward/std": 0.22074688971042633, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 96 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 832.0, |
| "completions/mean_length": 457.82293701171875, |
| "completions/mean_terminated_length": 449.66058349609375, |
| "completions/min_length": 227.0, |
| "completions/min_terminated_length": 227.0, |
| "entropy": 0.1109771728515625, |
| "epoch": 0.8850869689193043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.035750389099121094, |
| "kl": 0.0031118392944335938, |
| "learning_rate": 7.887634688515e-07, |
| "loss": -0.0005, |
| "num_tokens": 39751387.0, |
| "reward": 1.1470212936401367, |
| "reward_std": 0.19520321488380432, |
| "rewards/cross_solution_unittest_reward/mean": 0.5024754554033279, |
| "rewards/cross_solution_unittest_reward/std": 0.2238490730524063, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 97 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 856.0, |
| "completions/max_terminated_length": 856.0, |
| "completions/mean_length": 528.5989990234375, |
| "completions/mean_terminated_length": 528.5989990234375, |
| "completions/min_length": 285.0, |
| "completions/min_terminated_length": 285.0, |
| "entropy": 0.0987396240234375, |
| "epoch": 0.8942115768463074, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02971605211496353, |
| "kl": 0.002849102020263672, |
| "learning_rate": 6.813597078854772e-07, |
| "loss": 0.0069, |
| "num_tokens": 40197641.0, |
| "reward": 1.0776381492614746, |
| "reward_std": 0.2849399149417877, |
| "rewards/cross_solution_unittest_reward/mean": 0.4694649428129196, |
| "rewards/cross_solution_unittest_reward/std": 0.28404664993286133, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 911.0, |
| "completions/max_terminated_length": 911.0, |
| "completions/mean_length": 493.4036560058594, |
| "completions/mean_terminated_length": 493.4036560058594, |
| "completions/min_length": 209.0, |
| "completions/min_terminated_length": 209.0, |
| "entropy": 0.1012420654296875, |
| "epoch": 0.9033361847733106, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03130873292684555, |
| "kl": 0.0025815963745117188, |
| "learning_rate": 5.815563636047539e-07, |
| "loss": -0.0035, |
| "num_tokens": 40569268.0, |
| "reward": 1.0923125743865967, |
| "reward_std": 0.24533653259277344, |
| "rewards/cross_solution_unittest_reward/mean": 0.4748411625623703, |
| "rewards/cross_solution_unittest_reward/std": 0.2197420746088028, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 99 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 914.0, |
| "completions/max_terminated_length": 914.0, |
| "completions/mean_length": 474.8125, |
| "completions/mean_terminated_length": 474.8125, |
| "completions/min_length": 261.0, |
| "completions/min_terminated_length": 261.0, |
| "entropy": 0.1113739013671875, |
| "epoch": 0.9124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03394750878214836, |
| "kl": 0.003151416778564453, |
| "learning_rate": 4.894348370484648e-07, |
| "loss": 0.0001, |
| "num_tokens": 40993476.0, |
| "reward": 0.9635044932365417, |
| "reward_std": 0.24135762453079224, |
| "rewards/cross_solution_unittest_reward/mean": 0.4190831929445267, |
| "rewards/cross_solution_unittest_reward/std": 0.24994324147701263, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 848.0, |
| "completions/mean_length": 488.19012451171875, |
| "completions/mean_terminated_length": 480.1070556640625, |
| "completions/min_length": 248.0, |
| "completions/min_terminated_length": 248.0, |
| "entropy": 0.0947265625, |
| "epoch": 0.9215854006273168, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034001559019088745, |
| "kl": 0.003081798553466797, |
| "learning_rate": 4.0507026385502747e-07, |
| "loss": 0.0004, |
| "num_tokens": 41383509.0, |
| "reward": 1.047755479812622, |
| "reward_std": 0.23132114112377167, |
| "rewards/cross_solution_unittest_reward/mean": 0.4617375209927559, |
| "rewards/cross_solution_unittest_reward/std": 0.20788954198360443, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 101 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 995.0, |
| "completions/max_terminated_length": 995.0, |
| "completions/mean_length": 502.48699951171875, |
| "completions/mean_terminated_length": 502.48699951171875, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.094207763671875, |
| "epoch": 0.93071000855432, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028330322355031967, |
| "kl": 0.002512693405151367, |
| "learning_rate": 3.2853145298042954e-07, |
| "loss": -0.0035, |
| "num_tokens": 41811784.0, |
| "reward": 1.1942728757858276, |
| "reward_std": 0.28333157300949097, |
| "rewards/cross_solution_unittest_reward/mean": 0.5126944035291672, |
| "rewards/cross_solution_unittest_reward/std": 0.1964806318283081, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 968.0, |
| "completions/max_terminated_length": 968.0, |
| "completions/mean_length": 471.36199951171875, |
| "completions/mean_terminated_length": 471.36199951171875, |
| "completions/min_length": 241.0, |
| "completions/min_terminated_length": 241.0, |
| "entropy": 0.103363037109375, |
| "epoch": 0.939834616481323, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0329383909702301, |
| "kl": 0.0033469200134277344, |
| "learning_rate": 2.5988083057666534e-07, |
| "loss": -0.0076, |
| "num_tokens": 42183707.0, |
| "reward": 1.1951045989990234, |
| "reward_std": 0.2977498471736908, |
| "rewards/cross_solution_unittest_reward/mean": 0.514451876282692, |
| "rewards/cross_solution_unittest_reward/std": 0.251402385532856, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 103 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 866.0, |
| "completions/max_terminated_length": 866.0, |
| "completions/mean_length": 500.29168701171875, |
| "completions/mean_terminated_length": 500.29168701171875, |
| "completions/min_length": 287.0, |
| "completions/min_terminated_length": 287.0, |
| "entropy": 0.09759521484375, |
| "epoch": 0.9489592244083263, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0342080220580101, |
| "kl": 0.0033740997314453125, |
| "learning_rate": 1.9917438907606556e-07, |
| "loss": 0.0017, |
| "num_tokens": 42602043.0, |
| "reward": 1.004610300064087, |
| "reward_std": 0.2067084014415741, |
| "rewards/cross_solution_unittest_reward/mean": 0.44767728447914124, |
| "rewards/cross_solution_unittest_reward/std": 0.19676169008016586, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 104 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 942.0, |
| "completions/max_terminated_length": 942.0, |
| "completions/mean_length": 461.2161560058594, |
| "completions/mean_terminated_length": 461.2161560058594, |
| "completions/min_length": 275.0, |
| "completions/min_terminated_length": 275.0, |
| "entropy": 0.1075592041015625, |
| "epoch": 0.9580838323353293, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0386245995759964, |
| "kl": 0.0034677982330322266, |
| "learning_rate": 1.464616415230702e-07, |
| "loss": -0.0062, |
| "num_tokens": 42963262.0, |
| "reward": 0.8114880323410034, |
| "reward_std": 0.26597505807876587, |
| "rewards/cross_solution_unittest_reward/mean": 0.3618558570742607, |
| "rewards/cross_solution_unittest_reward/std": 0.23664288967847824, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 105 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 811.0, |
| "completions/max_terminated_length": 811.0, |
| "completions/mean_length": 524.5104370117188, |
| "completions/mean_terminated_length": 524.5104370117188, |
| "completions/min_length": 274.0, |
| "completions/min_terminated_length": 274.0, |
| "entropy": 0.093170166015625, |
| "epoch": 0.9672084402623324, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028961867094039917, |
| "kl": 0.0027861595153808594, |
| "learning_rate": 1.0178558119067316e-07, |
| "loss": -0.0063, |
| "num_tokens": 43390042.0, |
| "reward": 1.000401496887207, |
| "reward_std": 0.30445563793182373, |
| "rewards/cross_solution_unittest_reward/mean": 0.43068091571331024, |
| "rewards/cross_solution_unittest_reward/std": 0.25178908556699753, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 885.0, |
| "completions/max_terminated_length": 885.0, |
| "completions/mean_length": 494.6015625, |
| "completions/mean_terminated_length": 494.6015625, |
| "completions/min_length": 282.0, |
| "completions/min_terminated_length": 282.0, |
| "entropy": 0.1114959716796875, |
| "epoch": 0.9763330481893356, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034926630556583405, |
| "kl": 0.0031135082244873047, |
| "learning_rate": 6.51826465144978e-08, |
| "loss": -0.0033, |
| "num_tokens": 43794513.0, |
| "reward": 1.0083611011505127, |
| "reward_std": 0.23411409556865692, |
| "rewards/cross_solution_unittest_reward/mean": 0.4444109722971916, |
| "rewards/cross_solution_unittest_reward/std": 0.2296873927116394, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 107 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1069.0, |
| "completions/max_terminated_length": 1069.0, |
| "completions/mean_length": 530.9869995117188, |
| "completions/mean_terminated_length": 530.9869995117188, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.0896759033203125, |
| "epoch": 0.9854576561163387, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029170462861657143, |
| "kl": 0.002541780471801758, |
| "learning_rate": 3.668269137308666e-08, |
| "loss": 0.0042, |
| "num_tokens": 44236428.0, |
| "reward": 1.2101051807403564, |
| "reward_std": 0.225676029920578, |
| "rewards/cross_solution_unittest_reward/mean": 0.525607168674469, |
| "rewards/cross_solution_unittest_reward/std": 0.20895672589540482, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 108 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 841.0, |
| "completions/max_terminated_length": 841.0, |
| "completions/mean_length": 468.3489685058594, |
| "completions/mean_terminated_length": 468.3489685058594, |
| "completions/min_length": 216.0, |
| "completions/min_terminated_length": 216.0, |
| "entropy": 0.101654052734375, |
| "epoch": 0.9945822640433419, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03142247721552849, |
| "kl": 0.002844095230102539, |
| "learning_rate": 1.630896073864352e-08, |
| "loss": 0.0032, |
| "num_tokens": 44609178.0, |
| "reward": 1.2266151905059814, |
| "reward_std": 0.24263408780097961, |
| "rewards/cross_solution_unittest_reward/mean": 0.5332132428884506, |
| "rewards/cross_solution_unittest_reward/std": 0.22311152517795563, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.9945822640433419, |
| "step": 109, |
| "total_flos": 0.0, |
| "train_loss": 0.0030027573193699003, |
| "train_runtime": 43116.833, |
| "train_samples_per_second": 0.03, |
| "train_steps_per_second": 0.003 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 110, |
| "num_input_tokens_seen": 44609178, |
| "num_train_epochs": 1, |
| "save_steps": 25, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
