diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,3204 @@ +{ + "best_global_step": null, + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.9945822640433419, + "eval_steps": 500, + "global_step": 109, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 964.0, + "completions/max_terminated_length": 964.0, + "completions/mean_length": 527.6614990234375, + "completions/mean_terminated_length": 527.6614990234375, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.1096649169921875, + "epoch": 0.009124607927003137, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.023836389183998108, + "kl": 0.00021350383758544922, + "learning_rate": 2e-05, + "loss": -0.0088, + "num_tokens": 411230.0, + "reward": 1.1042730808258057, + "reward_std": 0.233424112200737, + "rewards/cross_solution_unittest_reward/mean": 0.481730118393898, + "rewards/cross_solution_unittest_reward/std": 0.22199106961488724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 1 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 920.0, + "completions/max_terminated_length": 920.0, + "completions/mean_length": 491.33856201171875, + "completions/mean_terminated_length": 491.33856201171875, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.1144561767578125, + "epoch": 0.018249215854006275, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029492009431123734, + "kl": 0.0002351701259613037, + "learning_rate": 1.9995921928281893e-05, + "loss": 0.0028, + "num_tokens": 821440.0, + "reward": 0.8423246741294861, + "reward_std": 0.2633346915245056, + "rewards/cross_solution_unittest_reward/mean": 0.371605783700943, + "rewards/cross_solution_unittest_reward/std": 0.25612012296915054, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 2 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1480.0, + "completions/max_terminated_length": 1480.0, + "completions/mean_length": 573.1614990234375, + "completions/mean_terminated_length": 573.1614990234375, + "completions/min_length": 295.0, + "completions/min_terminated_length": 295.0, + "entropy": 0.0967254638671875, + "epoch": 0.02737382378100941, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.026458488777279854, + "kl": 0.00021409988403320312, + "learning_rate": 1.9983691039261358e-05, + "loss": -0.0114, + "num_tokens": 1262302.0, + "reward": 0.9348706603050232, + "reward_std": 0.2640388309955597, + "rewards/cross_solution_unittest_reward/mean": 0.40714406967163086, + "rewards/cross_solution_unittest_reward/std": 0.22153814882040024, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 3 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 871.0, + "completions/max_terminated_length": 871.0, + "completions/mean_length": 506.05731201171875, + "completions/mean_terminated_length": 506.05731201171875, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.12152099609375, + "epoch": 0.03649843170801255, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028014499694108963, + "kl": 0.0002561211585998535, + "learning_rate": 1.9963317308626916e-05, + "loss": -0.0048, + "num_tokens": 1703844.0, + "reward": 0.6856307983398438, + "reward_std": 0.24917131662368774, + "rewards/cross_solution_unittest_reward/mean": 0.3077041059732437, + "rewards/cross_solution_unittest_reward/std": 0.18602629750967026, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 4 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1224.0, + "completions/max_terminated_length": 1224.0, + "completions/mean_length": 483.484375, + "completions/mean_terminated_length": 483.484375, + "completions/min_length": 247.0, + "completions/min_terminated_length": 247.0, + "entropy": 0.1254425048828125, + "epoch": 0.04562303963501568, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03355209901928902, + "kl": 0.0002625584602355957, + "learning_rate": 1.99348173534855e-05, + "loss": -0.004, + "num_tokens": 2084126.0, + "reward": 1.038306474685669, + "reward_std": 0.295562207698822, + "rewards/cross_solution_unittest_reward/mean": 0.4519632160663605, + "rewards/cross_solution_unittest_reward/std": 0.27628327906131744, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 5 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1174.0, + "completions/mean_length": 545.671875, + "completions/mean_terminated_length": 529.764404296875, + "completions/min_length": 293.0, + "completions/min_terminated_length": 293.0, + "entropy": 0.11370849609375, + "epoch": 0.05474764756201882, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.027173170819878578, + "kl": 0.00024318695068359375, + "learning_rate": 1.989821441880933e-05, + "loss": 0.0171, + "num_tokens": 2507912.0, + "reward": 0.9802842140197754, + "reward_std": 0.19230906665325165, + "rewards/cross_solution_unittest_reward/mean": 0.4297175034880638, + "rewards/cross_solution_unittest_reward/std": 0.24670374393463135, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 6 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.953125, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1065.0, + "completions/mean_length": 556.296875, + "completions/mean_terminated_length": 532.4566650390625, + "completions/min_length": 259.0, + "completions/min_terminated_length": 259.0, + "entropy": 0.1289520263671875, + "epoch": 0.06387225548902195, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02603895403444767, + "kl": 0.0002505183219909668, + "learning_rate": 1.9853538358476933e-05, + "loss": -0.0028, + "num_tokens": 2962034.0, + "reward": 0.9045166969299316, + "reward_std": 0.2844803035259247, + "rewards/cross_solution_unittest_reward/mean": 0.39844612032175064, + "rewards/cross_solution_unittest_reward/std": 0.21336368471384048, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 7 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1615.0, + "completions/mean_length": 534.5911865234375, + "completions/mean_terminated_length": 526.6292724609375, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.111663818359375, + "epoch": 0.0729968634160251, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02666051872074604, + "kl": 0.00025323033332824707, + "learning_rate": 1.9800825610923937e-05, + "loss": -0.0147, + "num_tokens": 3421973.0, + "reward": 0.8588270545005798, + "reward_std": 0.2657862901687622, + "rewards/cross_solution_unittest_reward/mean": 0.37082912772893906, + "rewards/cross_solution_unittest_reward/std": 0.2715388759970665, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 8 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1115.0, + "completions/max_terminated_length": 1115.0, + "completions/mean_length": 489.3724060058594, + "completions/mean_terminated_length": 489.3724060058594, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.122955322265625, + "epoch": 0.08212147134302823, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02615615725517273, + "kl": 0.0002747476100921631, + "learning_rate": 1.9740119169423337e-05, + "loss": 0.0001, + "num_tokens": 3813332.0, + "reward": 0.886775016784668, + "reward_std": 0.2901659309864044, + "rewards/cross_solution_unittest_reward/mean": 0.39264198392629623, + "rewards/cross_solution_unittest_reward/std": 0.22153189778327942, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 9 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 908.0, + "completions/max_terminated_length": 908.0, + "completions/mean_length": 511.34375, + "completions/mean_terminated_length": 511.34375, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "entropy": 0.1106414794921875, + "epoch": 0.09124607927003137, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.027914274483919144, + "kl": 0.0002880394458770752, + "learning_rate": 1.9671468547019575e-05, + "loss": 0.0019, + "num_tokens": 4259928.0, + "reward": 0.8995641469955444, + "reward_std": 0.2562140226364136, + "rewards/cross_solution_unittest_reward/mean": 0.395971842110157, + "rewards/cross_solution_unittest_reward/std": 0.23649413138628006, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 10 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1339.0, + "completions/max_terminated_length": 1339.0, + "completions/mean_length": 548.9244995117188, + "completions/mean_terminated_length": 548.9244995117188, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.1057586669921875, + "epoch": 0.1003706871970345, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028584128245711327, + "kl": 0.0003082454204559326, + "learning_rate": 1.9594929736144978e-05, + "loss": 0.0015, + "num_tokens": 4686539.0, + "reward": 0.9073726534843445, + "reward_std": 0.2739028036594391, + "rewards/cross_solution_unittest_reward/mean": 0.40253733843564987, + "rewards/cross_solution_unittest_reward/std": 0.2085171639919281, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 11 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1507.0, + "completions/max_terminated_length": 1507.0, + "completions/mean_length": 504.3645935058594, + "completions/mean_terminated_length": 504.3645935058594, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.1241607666015625, + "epoch": 0.10949529512403763, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03190146014094353, + "kl": 0.00035694241523742676, + "learning_rate": 1.9510565162951538e-05, + "loss": -0.0026, + "num_tokens": 5118263.0, + "reward": 0.7852699160575867, + "reward_std": 0.29856932163238525, + "rewards/cross_solution_unittest_reward/mean": 0.3413052186369896, + "rewards/cross_solution_unittest_reward/std": 0.2405463457107544, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 12 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1108.0, + "completions/mean_length": 537.6640625, + "completions/mean_terminated_length": 529.710205078125, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.1090240478515625, + "epoch": 0.11861990305104078, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03350791707634926, + "kl": 0.0005662441253662109, + "learning_rate": 1.941844363639525e-05, + "loss": -0.0074, + "num_tokens": 5532270.0, + "reward": 0.9594830870628357, + "reward_std": 0.22333046793937683, + "rewards/cross_solution_unittest_reward/mean": 0.42083391547203064, + "rewards/cross_solution_unittest_reward/std": 0.24756141006946564, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 13 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 2111.0, + "completions/max_terminated_length": 2111.0, + "completions/mean_length": 610.9375, + "completions/mean_terminated_length": 610.9375, + "completions/min_length": 245.0, + "completions/min_terminated_length": 245.0, + "entropy": 0.10589599609375, + "epoch": 0.1277445109780439, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028112368658185005, + "kl": 0.00034502148628234863, + "learning_rate": 1.9318640292114526e-05, + "loss": -0.0104, + "num_tokens": 5998974.0, + "reward": 1.0141568183898926, + "reward_std": 0.2032373547554016, + "rewards/cross_solution_unittest_reward/mean": 0.44281716644763947, + "rewards/cross_solution_unittest_reward/std": 0.2426743358373642, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 14 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 858.0, + "completions/max_terminated_length": 858.0, + "completions/mean_length": 463.62762451171875, + "completions/mean_terminated_length": 463.62762451171875, + "completions/min_length": 226.0, + "completions/min_terminated_length": 226.0, + "entropy": 0.12762451171875, + "epoch": 0.13686911890504705, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029718179255723953, + "kl": 0.00044339895248413086, + "learning_rate": 1.92112365311485e-05, + "loss": 0.0141, + "num_tokens": 6343095.0, + "reward": 1.0152561664581299, + "reward_std": 0.24558740854263306, + "rewards/cross_solution_unittest_reward/mean": 0.4407452195882797, + "rewards/cross_solution_unittest_reward/std": 0.2384948655962944, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 15 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.953125, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 3444.0, + "completions/mean_length": 578.6536865234375, + "completions/mean_terminated_length": 554.989501953125, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.1071319580078125, + "epoch": 0.1459937268320502, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028195451945066452, + "kl": 0.00040775537490844727, + "learning_rate": 1.9096319953545186e-05, + "loss": 0.0066, + "num_tokens": 6764442.0, + "reward": 0.8980576992034912, + "reward_std": 0.28982824087142944, + "rewards/cross_solution_unittest_reward/mean": 0.3970365896821022, + "rewards/cross_solution_unittest_reward/std": 0.2071005031466484, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 16 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 2372.0, + "completions/mean_length": 528.796875, + "completions/mean_terminated_length": 520.81982421875, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.11444091796875, + "epoch": 0.15511833475905332, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030508190393447876, + "kl": 0.00042378902435302734, + "learning_rate": 1.8973984286913584e-05, + "loss": 0.0154, + "num_tokens": 7181420.0, + "reward": 1.1027473211288452, + "reward_std": 0.28832367062568665, + "rewards/cross_solution_unittest_reward/mean": 0.46901851892471313, + "rewards/cross_solution_unittest_reward/std": 0.2841140776872635, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 17 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1141.0, + "completions/max_terminated_length": 1141.0, + "completions/mean_length": 544.4661865234375, + "completions/mean_terminated_length": 544.4661865234375, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.104888916015625, + "epoch": 0.16424294268605646, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028670065104961395, + "kl": 0.00043770670890808105, + "learning_rate": 1.8844329309978146e-05, + "loss": 0.002, + "num_tokens": 7620935.0, + "reward": 0.9763619303703308, + "reward_std": 0.243991419672966, + "rewards/cross_solution_unittest_reward/mean": 0.4205486625432968, + "rewards/cross_solution_unittest_reward/std": 0.23177453130483627, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 18 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1056.0, + "completions/max_terminated_length": 1056.0, + "completions/mean_length": 503.703125, + "completions/mean_terminated_length": 503.703125, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.113372802734375, + "epoch": 0.17336755061305958, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02729717455804348, + "kl": 0.0005033016204833984, + "learning_rate": 1.8707460771197773e-05, + "loss": 0.0014, + "num_tokens": 8032845.0, + "reward": 0.9106826782226562, + "reward_std": 0.27397438883781433, + "rewards/cross_solution_unittest_reward/mean": 0.39500948786735535, + "rewards/cross_solution_unittest_reward/std": 0.2559768706560135, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 19 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 960.0, + "completions/max_terminated_length": 960.0, + "completions/mean_length": 545.7396240234375, + "completions/mean_terminated_length": 545.7396240234375, + "completions/min_length": 322.0, + "completions/min_terminated_length": 322.0, + "entropy": 0.1036376953125, + "epoch": 0.18249215854006273, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.027323100715875626, + "kl": 0.0005331039428710938, + "learning_rate": 1.856349030251589e-05, + "loss": 0.0049, + "num_tokens": 8492777.0, + "reward": 0.9297716021537781, + "reward_std": 0.29434794187545776, + "rewards/cross_solution_unittest_reward/mean": 0.4003482013940811, + "rewards/cross_solution_unittest_reward/std": 0.2353406548500061, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 20 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1354.0, + "completions/max_terminated_length": 1354.0, + "completions/mean_length": 492.17449951171875, + "completions/mean_terminated_length": 492.17449951171875, + "completions/min_length": 244.0, + "completions/min_terminated_length": 244.0, + "entropy": 0.105987548828125, + "epoch": 0.19161676646706588, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02947384864091873, + "kl": 0.0005356073379516602, + "learning_rate": 1.8412535328311813e-05, + "loss": 0.006, + "num_tokens": 8880476.0, + "reward": 0.9555484652519226, + "reward_std": 0.23427900671958923, + "rewards/cross_solution_unittest_reward/mean": 0.4216877445578575, + "rewards/cross_solution_unittest_reward/std": 0.21247681975364685, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 21 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1018.0, + "completions/max_terminated_length": 1018.0, + "completions/mean_length": 506.7552185058594, + "completions/mean_terminated_length": 506.7552185058594, + "completions/min_length": 218.0, + "completions/min_terminated_length": 218.0, + "entropy": 0.1118011474609375, + "epoch": 0.200741374394069, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03192589804530144, + "kl": 0.0005907416343688965, + "learning_rate": 1.825471896962774e-05, + "loss": -0.0137, + "num_tokens": 9256446.0, + "reward": 0.9712496995925903, + "reward_std": 0.2654087245464325, + "rewards/cross_solution_unittest_reward/mean": 0.41771475970745087, + "rewards/cross_solution_unittest_reward/std": 0.2567882165312767, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 22 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1568.0, + "completions/max_terminated_length": 1568.0, + "completions/mean_length": 527.6666870117188, + "completions/mean_terminated_length": 527.6666870117188, + "completions/min_length": 289.0, + "completions/min_terminated_length": 289.0, + "entropy": 0.108154296875, + "epoch": 0.20986598232107215, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02674136683344841, + "kl": 0.000629425048828125, + "learning_rate": 1.8090169943749477e-05, + "loss": -0.0106, + "num_tokens": 9710614.0, + "reward": 1.0353918075561523, + "reward_std": 0.22365829348564148, + "rewards/cross_solution_unittest_reward/mean": 0.4482730031013489, + "rewards/cross_solution_unittest_reward/std": 0.2228257954120636, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 23 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 834.0, + "completions/max_terminated_length": 834.0, + "completions/mean_length": 509.2395935058594, + "completions/mean_terminated_length": 509.2395935058594, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.1023101806640625, + "epoch": 0.21899059024807527, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02758539468050003, + "kl": 0.0006355643272399902, + "learning_rate": 1.7919022459222754e-05, + "loss": 0.0141, + "num_tokens": 10133834.0, + "reward": 0.8483362197875977, + "reward_std": 0.2694234549999237, + "rewards/cross_solution_unittest_reward/mean": 0.3766419366002083, + "rewards/cross_solution_unittest_reward/std": 0.26132430136203766, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 24 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 980.0, + "completions/mean_length": 494.6380310058594, + "completions/mean_terminated_length": 486.5718078613281, + "completions/min_length": 231.0, + "completions/min_terminated_length": 231.0, + "entropy": 0.11871337890625, + "epoch": 0.22811519817507842, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0348508358001709, + "kl": 0.0009682774543762207, + "learning_rate": 1.7741416106390828e-05, + "loss": -0.0021, + "num_tokens": 10537447.0, + "reward": 0.9557062387466431, + "reward_std": 0.22823719680309296, + "rewards/cross_solution_unittest_reward/mean": 0.4154369905591011, + "rewards/cross_solution_unittest_reward/std": 0.20701014250516891, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 25 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1757.0, + "completions/max_terminated_length": 1757.0, + "completions/mean_length": 497.625, + "completions/mean_terminated_length": 497.625, + "completions/min_length": 213.0, + "completions/min_terminated_length": 213.0, + "entropy": 0.11309814453125, + "epoch": 0.23723980610208156, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028710953891277313, + "kl": 0.0007503032684326172, + "learning_rate": 1.7557495743542586e-05, + "loss": 0.0115, + "num_tokens": 10895703.0, + "reward": 0.8731624484062195, + "reward_std": 0.24689321219921112, + "rewards/cross_solution_unittest_reward/mean": 0.3826525807380676, + "rewards/cross_solution_unittest_reward/std": 0.24787958711385727, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 26 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 902.0, + "completions/max_terminated_length": 902.0, + "completions/mean_length": 473.3671875, + "completions/mean_terminated_length": 473.3671875, + "completions/min_length": 276.0, + "completions/min_terminated_length": 276.0, + "entropy": 0.111358642578125, + "epoch": 0.24636441402908468, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030619844794273376, + "kl": 0.0008200407028198242, + "learning_rate": 1.736741137876405e-05, + "loss": 0.002, + "num_tokens": 11274364.0, + "reward": 1.0031293630599976, + "reward_std": 0.2526843547821045, + "rewards/cross_solution_unittest_reward/mean": 0.44217514991760254, + "rewards/cross_solution_unittest_reward/std": 0.21224447339773178, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 27 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 957.0, + "completions/max_terminated_length": 957.0, + "completions/mean_length": 506.1458435058594, + "completions/mean_terminated_length": 506.1458435058594, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.1076812744140625, + "epoch": 0.2554890219560878, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.026609111577272415, + "kl": 0.0008921623229980469, + "learning_rate": 1.7171318047589637e-05, + "loss": -0.0014, + "num_tokens": 11706916.0, + "reward": 1.137014389038086, + "reward_std": 0.2074248045682907, + "rewards/cross_solution_unittest_reward/mean": 0.4929664433002472, + "rewards/cross_solution_unittest_reward/std": 0.26993829011917114, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 28 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 952.0, + "completions/max_terminated_length": 952.0, + "completions/mean_length": 489.57293701171875, + "completions/mean_terminated_length": 489.57293701171875, + "completions/min_length": 256.0, + "completions/min_terminated_length": 256.0, + "entropy": 0.117828369140625, + "epoch": 0.26461362988309095, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.031492725014686584, + "kl": 0.001159071922302246, + "learning_rate": 1.696937568655294e-05, + "loss": -0.0028, + "num_tokens": 12102904.0, + "reward": 1.0333976745605469, + "reward_std": 0.2310364693403244, + "rewards/cross_solution_unittest_reward/mean": 0.4447094053030014, + "rewards/cross_solution_unittest_reward/std": 0.29005488753318787, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 29 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1168.0, + "completions/mean_length": 544.9635620117188, + "completions/mean_terminated_length": 529.0523681640625, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.103607177734375, + "epoch": 0.2737382378100941, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028927048668265343, + "kl": 0.0009303092956542969, + "learning_rate": 1.6761749002740195e-05, + "loss": 0.0036, + "num_tokens": 12531050.0, + "reward": 0.8030904531478882, + "reward_std": 0.2894265055656433, + "rewards/cross_solution_unittest_reward/mean": 0.35538169741630554, + "rewards/cross_solution_unittest_reward/std": 0.27038178592920303, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 30 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 946.0, + "completions/mean_length": 486.05731201171875, + "completions/mean_terminated_length": 469.83770751953125, + "completions/min_length": 236.0, + "completions/min_terminated_length": 236.0, + "entropy": 0.1143035888671875, + "epoch": 0.28286284573709725, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030626529827713966, + "kl": 0.0011305809020996094, + "learning_rate": 1.6548607339452853e-05, + "loss": 0.0161, + "num_tokens": 12901128.0, + "reward": 1.0075210332870483, + "reward_std": 0.2209041714668274, + "rewards/cross_solution_unittest_reward/mean": 0.4487333372235298, + "rewards/cross_solution_unittest_reward/std": 0.21068686991930008, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 31 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 954.0, + "completions/max_terminated_length": 954.0, + "completions/mean_length": 481.5677185058594, + "completions/mean_terminated_length": 481.5677185058594, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.103546142578125, + "epoch": 0.2919874536641004, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03601611778140068, + "kl": 0.0013216733932495117, + "learning_rate": 1.6330124538088705e-05, + "loss": 0.0017, + "num_tokens": 13354434.0, + "reward": 1.051571011543274, + "reward_std": 0.3614152669906616, + "rewards/cross_solution_unittest_reward/mean": 0.4522690027952194, + "rewards/cross_solution_unittest_reward/std": 0.24180202186107635, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 32 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 773.0, + "completions/max_terminated_length": 773.0, + "completions/mean_length": 469.72137451171875, + "completions/mean_terminated_length": 469.72137451171875, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.115203857421875, + "epoch": 0.3011120615911035, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02829568460583687, + "kl": 0.0014170408248901367, + "learning_rate": 1.6106478796354382e-05, + "loss": 0.0034, + "num_tokens": 13732783.0, + "reward": 0.921411395072937, + "reward_std": 0.17318974435329437, + "rewards/cross_solution_unittest_reward/mean": 0.4217027798295021, + "rewards/cross_solution_unittest_reward/std": 0.22807485610246658, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 33 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 839.0, + "completions/max_terminated_length": 839.0, + "completions/mean_length": 488.7578125, + "completions/mean_terminated_length": 488.7578125, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.1075286865234375, + "epoch": 0.31023666951810663, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029855074360966682, + "kl": 0.001324295997619629, + "learning_rate": 1.5877852522924733e-05, + "loss": -0.0055, + "num_tokens": 14159730.0, + "reward": 1.081244707107544, + "reward_std": 0.18814358115196228, + "rewards/cross_solution_unittest_reward/mean": 0.4768352806568146, + "rewards/cross_solution_unittest_reward/std": 0.22413263469934464, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 34 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1815.0, + "completions/max_terminated_length": 1815.0, + "completions/mean_length": 510.74481201171875, + "completions/mean_terminated_length": 510.74481201171875, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.1134490966796875, + "epoch": 0.3193612774451098, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03141755983233452, + "kl": 0.0015311241149902344, + "learning_rate": 1.5644432188667695e-05, + "loss": 0.0011, + "num_tokens": 14547176.0, + "reward": 0.9383748769760132, + "reward_std": 0.2669951617717743, + "rewards/cross_solution_unittest_reward/mean": 0.3990442007780075, + "rewards/cross_solution_unittest_reward/std": 0.24042869359254837, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 35 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1357.0, + "completions/mean_length": 537.3489990234375, + "completions/mean_terminated_length": 529.394287109375, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.1122283935546875, + "epoch": 0.32848588537211293, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030957432463765144, + "kl": 0.0015711784362792969, + "learning_rate": 1.5406408174555978e-05, + "loss": -0.0053, + "num_tokens": 14975526.0, + "reward": 0.9109244346618652, + "reward_std": 0.25726160407066345, + "rewards/cross_solution_unittest_reward/mean": 0.3915872722864151, + "rewards/cross_solution_unittest_reward/std": 0.22853311151266098, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 36 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 2463.0, + "completions/mean_length": 520.1276245117188, + "completions/mean_terminated_length": 512.1279296875, + "completions/min_length": 246.0, + "completions/min_terminated_length": 246.0, + "entropy": 0.097320556640625, + "epoch": 0.3376104932991161, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.032772429287433624, + "kl": 0.0015528202056884766, + "learning_rate": 1.5163974616389621e-05, + "loss": 0.0123, + "num_tokens": 15386143.0, + "reward": 0.9813830256462097, + "reward_std": 0.22931739687919617, + "rewards/cross_solution_unittest_reward/mean": 0.43425410240888596, + "rewards/cross_solution_unittest_reward/std": 0.22800295799970627, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 37 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 887.0, + "completions/max_terminated_length": 887.0, + "completions/mean_length": 485.6380310058594, + "completions/mean_terminated_length": 485.6380310058594, + "completions/min_length": 205.0, + "completions/min_terminated_length": 205.0, + "entropy": 0.1133270263671875, + "epoch": 0.34673510122611917, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03404747322201729, + "kl": 0.0015780925750732422, + "learning_rate": 1.491732924645604e-05, + "loss": 0.0191, + "num_tokens": 15770812.0, + "reward": 0.9875783920288086, + "reward_std": 0.2382659763097763, + "rewards/cross_solution_unittest_reward/mean": 0.4320690929889679, + "rewards/cross_solution_unittest_reward/std": 0.2256365790963173, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 38 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1240.0, + "completions/max_terminated_length": 1240.0, + "completions/mean_length": 473.8984375, + "completions/mean_terminated_length": 473.8984375, + "completions/min_length": 179.0, + "completions/min_terminated_length": 179.0, + "entropy": 0.116729736328125, + "epoch": 0.3558597091531223, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03682932257652283, + "kl": 0.003524303436279297, + "learning_rate": 1.4666673232256738e-05, + "loss": -0.0084, + "num_tokens": 16082925.0, + "reward": 0.8647634387016296, + "reward_std": 0.3210960626602173, + "rewards/cross_solution_unittest_reward/mean": 0.37845826894044876, + "rewards/cross_solution_unittest_reward/std": 0.2232651561498642, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 39 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 2045.0, + "completions/max_terminated_length": 2045.0, + "completions/mean_length": 503.125, + "completions/mean_terminated_length": 503.125, + "completions/min_length": 202.0, + "completions/min_terminated_length": 202.0, + "entropy": 0.102325439453125, + "epoch": 0.36498431708012546, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.04535383731126785, + "kl": 0.003467082977294922, + "learning_rate": 1.4412211012432213e-05, + "loss": 0.0068, + "num_tokens": 16496789.0, + "reward": 0.8126609325408936, + "reward_std": 0.26924118399620056, + "rewards/cross_solution_unittest_reward/mean": 0.3569253906607628, + "rewards/cross_solution_unittest_reward/std": 0.2864082232117653, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 40 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 958.0, + "completions/max_terminated_length": 958.0, + "completions/mean_length": 511.31512451171875, + "completions/mean_terminated_length": 511.31512451171875, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.1136627197265625, + "epoch": 0.3741089250071286, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033672649413347244, + "kl": 0.0028324127197265625, + "learning_rate": 1.4154150130018867e-05, + "loss": -0.0017, + "num_tokens": 16912990.0, + "reward": 0.7562817335128784, + "reward_std": 0.20014888048171997, + "rewards/cross_solution_unittest_reward/mean": 0.3420030549168587, + "rewards/cross_solution_unittest_reward/std": 0.21658644825220108, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 41 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1215.0, + "completions/max_terminated_length": 1215.0, + "completions/mean_length": 506.51043701171875, + "completions/mean_terminated_length": 506.51043701171875, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.093170166015625, + "epoch": 0.38323353293413176, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03319159895181656, + "kl": 0.0018248558044433594, + "learning_rate": 1.3892701063173917e-05, + "loss": -0.0041, + "num_tokens": 17357914.0, + "reward": 0.9257922172546387, + "reward_std": 0.2104812115430832, + "rewards/cross_solution_unittest_reward/mean": 0.4091387465596199, + "rewards/cross_solution_unittest_reward/std": 0.23963715881109238, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 42 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 799.0, + "completions/max_terminated_length": 799.0, + "completions/mean_length": 478.70574951171875, + "completions/mean_terminated_length": 478.70574951171875, + "completions/min_length": 249.0, + "completions/min_terminated_length": 249.0, + "entropy": 0.105377197265625, + "epoch": 0.39235814086113485, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03309491649270058, + "kl": 0.00234222412109375, + "learning_rate": 1.362807705350641e-05, + "loss": -0.0027, + "num_tokens": 17696089.0, + "reward": 1.0697866678237915, + "reward_std": 0.21351736783981323, + "rewards/cross_solution_unittest_reward/mean": 0.46968914568424225, + "rewards/cross_solution_unittest_reward/std": 0.25238344073295593, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 43 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 2798.0, + "completions/mean_length": 519.0208740234375, + "completions/mean_terminated_length": 511.0182800292969, + "completions/min_length": 224.0, + "completions/min_terminated_length": 224.0, + "entropy": 0.095550537109375, + "epoch": 0.401482748788138, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029689131304621696, + "kl": 0.0020627975463867188, + "learning_rate": 1.3360493932154301e-05, + "loss": -0.0021, + "num_tokens": 18106297.0, + "reward": 0.7189862728118896, + "reward_std": 0.21532303094863892, + "rewards/cross_solution_unittest_reward/mean": 0.3341630734503269, + "rewards/cross_solution_unittest_reward/std": 0.210732139647007, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 44 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1472.0, + "completions/max_terminated_length": 1472.0, + "completions/mean_length": 467.91668701171875, + "completions/mean_terminated_length": 467.91668701171875, + "completions/min_length": 196.0, + "completions/min_terminated_length": 196.0, + "entropy": 0.1134033203125, + "epoch": 0.41060735671514115, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033839646726846695, + "kl": 0.0030853748321533203, + "learning_rate": 1.3090169943749475e-05, + "loss": 0.021, + "num_tokens": 18488529.0, + "reward": 1.0240888595581055, + "reward_std": 0.21602611243724823, + "rewards/cross_solution_unittest_reward/mean": 0.4490320533514023, + "rewards/cross_solution_unittest_reward/std": 0.2549639642238617, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 45 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1161.0, + "completions/mean_length": 520.1119995117188, + "completions/mean_terminated_length": 512.1123046875, + "completions/min_length": 281.0, + "completions/min_terminated_length": 281.0, + "entropy": 0.0985565185546875, + "epoch": 0.4197319646421443, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.031324755400419235, + "kl": 0.0023124217987060547, + "learning_rate": 1.2817325568414299e-05, + "loss": -0.0063, + "num_tokens": 18908484.0, + "reward": 0.8984124064445496, + "reward_std": 0.31652843952178955, + "rewards/cross_solution_unittest_reward/mean": 0.394581638276577, + "rewards/cross_solution_unittest_reward/std": 0.2338028848171234, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 46 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.921875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1145.0, + "completions/mean_length": 528.3125, + "completions/mean_terminated_length": 488.0000305175781, + "completions/min_length": 215.0, + "completions/min_terminated_length": 215.0, + "entropy": 0.11846923828125, + "epoch": 0.42885657256914744, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03654487058520317, + "kl": 0.003631591796875, + "learning_rate": 1.2542183341934873e-05, + "loss": 0.0342, + "num_tokens": 19336532.0, + "reward": 0.7277412414550781, + "reward_std": 0.2056652158498764, + "rewards/cross_solution_unittest_reward/mean": 0.3252406492829323, + "rewards/cross_solution_unittest_reward/std": 0.24651427567005157, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 47 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1054.0, + "completions/max_terminated_length": 1054.0, + "completions/mean_length": 492.11199951171875, + "completions/mean_terminated_length": 492.11199951171875, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.1041717529296875, + "epoch": 0.43798118049615054, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03339504823088646, + "kl": 0.0033910274505615234, + "learning_rate": 1.2264967674257647e-05, + "loss": 0.0055, + "num_tokens": 19727471.0, + "reward": 0.9818544387817383, + "reward_std": 0.26466554403305054, + "rewards/cross_solution_unittest_reward/mean": 0.4283819645643234, + "rewards/cross_solution_unittest_reward/std": 0.22476153075695038, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 48 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1095.0, + "completions/mean_length": 502.90625, + "completions/mean_terminated_length": 494.86163330078125, + "completions/min_length": 262.0, + "completions/min_terminated_length": 262.0, + "entropy": 0.1012725830078125, + "epoch": 0.4471057884231537, + "frac_reward_zero_std": 0.0, + "grad_norm": 18.39661979675293, + "kl": 0.8277513980865479, + "learning_rate": 1.1985904666457455e-05, + "loss": 0.0468, + "num_tokens": 20142435.0, + "reward": 0.7730751037597656, + "reward_std": 0.30047422647476196, + "rewards/cross_solution_unittest_reward/mean": 0.33403923362493515, + "rewards/cross_solution_unittest_reward/std": 0.25003235042095184, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 49 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1312.0, + "completions/max_terminated_length": 1312.0, + "completions/mean_length": 509.84375, + "completions/mean_terminated_length": 509.84375, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.09967041015625, + "epoch": 0.45623039635015683, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03424382582306862, + "kl": 0.0036292076110839844, + "learning_rate": 1.170522192632624e-05, + "loss": -0.0109, + "num_tokens": 20554295.0, + "reward": 0.9384050369262695, + "reward_std": 0.2790818512439728, + "rewards/cross_solution_unittest_reward/mean": 0.41122280806303024, + "rewards/cross_solution_unittest_reward/std": 0.28970006108283997, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 50 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 909.0, + "completions/max_terminated_length": 909.0, + "completions/mean_length": 509.796875, + "completions/mean_terminated_length": 509.796875, + "completions/min_length": 173.0, + "completions/min_terminated_length": 173.0, + "entropy": 0.0945892333984375, + "epoch": 0.46535500427716, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030270231887698174, + "kl": 0.003202676773071289, + "learning_rate": 1.1423148382732854e-05, + "loss": 0.0027, + "num_tokens": 20974745.0, + "reward": 1.0798063278198242, + "reward_std": 0.2213771790266037, + "rewards/cross_solution_unittest_reward/mean": 0.4737387150526047, + "rewards/cross_solution_unittest_reward/std": 0.18860329687595367, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 51 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1354.0, + "completions/mean_length": 513.109375, + "completions/mean_terminated_length": 505.0914001464844, + "completions/min_length": 251.0, + "completions/min_terminated_length": 251.0, + "entropy": 0.099945068359375, + "epoch": 0.4744796122041631, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03432578966021538, + "kl": 0.0032448768615722656, + "learning_rate": 1.1139914098905406e-05, + "loss": 0.0178, + "num_tokens": 21378419.0, + "reward": 1.0716464519500732, + "reward_std": 0.2842645049095154, + "rewards/cross_solution_unittest_reward/mean": 0.47282470762729645, + "rewards/cross_solution_unittest_reward/std": 0.23804496228694916, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 52 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1391.0, + "completions/max_terminated_length": 1391.0, + "completions/mean_length": 487.3828125, + "completions/mean_terminated_length": 487.3828125, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.0956878662109375, + "epoch": 0.4836042201311662, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03196337819099426, + "kl": 0.0026366710662841797, + "learning_rate": 1.08557500847884e-05, + "loss": -0.0004, + "num_tokens": 21779142.0, + "reward": 0.8248944282531738, + "reward_std": 0.2299119085073471, + "rewards/cross_solution_unittest_reward/mean": 0.3687998875975609, + "rewards/cross_solution_unittest_reward/std": 0.22956574708223343, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 53 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1238.0, + "completions/max_terminated_length": 1238.0, + "completions/mean_length": 536.3932495117188, + "completions/mean_terminated_length": 536.3932495117188, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.1009521484375, + "epoch": 0.49272882805816937, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03287142142653465, + "kl": 0.003429412841796875, + "learning_rate": 1.0570888108627682e-05, + "loss": -0.0033, + "num_tokens": 22241477.0, + "reward": 0.8158465623855591, + "reward_std": 0.2910717725753784, + "rewards/cross_solution_unittest_reward/mean": 0.358261875808239, + "rewards/cross_solution_unittest_reward/std": 0.21051427721977234, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 54 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 898.0, + "completions/max_terminated_length": 898.0, + "completions/mean_length": 491.3802185058594, + "completions/mean_terminated_length": 491.3802185058594, + "completions/min_length": 219.0, + "completions/min_terminated_length": 219.0, + "entropy": 0.1052398681640625, + "epoch": 0.5018534359851725, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03173370659351349, + "kl": 0.0028574466705322266, + "learning_rate": 1.0285560507936962e-05, + "loss": -0.0091, + "num_tokens": 22619007.0, + "reward": 0.8488973379135132, + "reward_std": 0.24665257334709167, + "rewards/cross_solution_unittest_reward/mean": 0.37161923199892044, + "rewards/cross_solution_unittest_reward/std": 0.21222124248743057, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 55 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 932.0, + "completions/max_terminated_length": 932.0, + "completions/mean_length": 494.5208435058594, + "completions/mean_terminated_length": 494.5208435058594, + "completions/min_length": 264.0, + "completions/min_terminated_length": 264.0, + "entropy": 0.1017913818359375, + "epoch": 0.5109780439121756, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.045224472880363464, + "kl": 0.00625157356262207, + "learning_rate": 1e-05, + "loss": 0.0093, + "num_tokens": 22993871.0, + "reward": 0.9006222486495972, + "reward_std": 0.21599134802818298, + "rewards/cross_solution_unittest_reward/mean": 0.40076694637537, + "rewards/cross_solution_unittest_reward/std": 0.22537627071142197, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 56 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 981.0, + "completions/mean_length": 510.2161560058594, + "completions/mean_terminated_length": 502.19061279296875, + "completions/min_length": 308.0, + "completions/min_terminated_length": 308.0, + "entropy": 0.1026153564453125, + "epoch": 0.5201026518391788, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033613454550504684, + "kl": 0.0032994747161865234, + "learning_rate": 9.71443949206304e-06, + "loss": -0.0064, + "num_tokens": 23442418.0, + "reward": 0.9552373290061951, + "reward_std": 0.21273988485336304, + "rewards/cross_solution_unittest_reward/mean": 0.4271874502301216, + "rewards/cross_solution_unittest_reward/std": 0.20565658062696457, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 57 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 925.0, + "completions/mean_length": 494.5208435058594, + "completions/mean_terminated_length": 478.3455505371094, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.117401123046875, + "epoch": 0.5292272597661819, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03614190220832825, + "kl": 0.0036954879760742188, + "learning_rate": 9.42911189137232e-06, + "loss": 0.0155, + "num_tokens": 23857530.0, + "reward": 0.9477154016494751, + "reward_std": 0.20985114574432373, + "rewards/cross_solution_unittest_reward/mean": 0.41357675194740295, + "rewards/cross_solution_unittest_reward/std": 0.2724653407931328, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 58 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 899.0, + "completions/mean_length": 474.82293701171875, + "completions/mean_terminated_length": 466.7049560546875, + "completions/min_length": 242.0, + "completions/min_terminated_length": 242.0, + "entropy": 0.1109161376953125, + "epoch": 0.5383518676931851, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03441567346453667, + "kl": 0.00311279296875, + "learning_rate": 9.144249915211605e-06, + "loss": 0.0062, + "num_tokens": 24261294.0, + "reward": 0.9669393301010132, + "reward_std": 0.22922192513942719, + "rewards/cross_solution_unittest_reward/mean": 0.4128674417734146, + "rewards/cross_solution_unittest_reward/std": 0.2560960501432419, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 59 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 854.0, + "completions/max_terminated_length": 854.0, + "completions/mean_length": 497.9609375, + "completions/mean_terminated_length": 497.9609375, + "completions/min_length": 250.0, + "completions/min_terminated_length": 250.0, + "entropy": 0.0975494384765625, + "epoch": 0.5474764756201882, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033020906150341034, + "kl": 0.002737283706665039, + "learning_rate": 8.860085901094595e-06, + "loss": -0.0018, + "num_tokens": 24696543.0, + "reward": 0.9655224680900574, + "reward_std": 0.28828054666519165, + "rewards/cross_solution_unittest_reward/mean": 0.4253334328532219, + "rewards/cross_solution_unittest_reward/std": 0.23384006321430206, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 60 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 999.0, + "completions/max_terminated_length": 999.0, + "completions/mean_length": 524.890625, + "completions/mean_terminated_length": 524.890625, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.096435546875, + "epoch": 0.5566010835471913, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03234833851456642, + "kl": 0.003005504608154297, + "learning_rate": 8.576851617267151e-06, + "loss": 0.0086, + "num_tokens": 25143405.0, + "reward": 1.010263442993164, + "reward_std": 0.23946420848369598, + "rewards/cross_solution_unittest_reward/mean": 0.44504720717668533, + "rewards/cross_solution_unittest_reward/std": 0.2548332065343857, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 61 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1025.0, + "completions/mean_length": 530.8958740234375, + "completions/mean_terminated_length": 522.92431640625, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.10467529296875, + "epoch": 0.5657256914741945, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03223549574613571, + "kl": 0.002941131591796875, + "learning_rate": 8.294778073673762e-06, + "loss": 0.007, + "num_tokens": 25572309.0, + "reward": 0.7772404551506042, + "reward_std": 0.2317599058151245, + "rewards/cross_solution_unittest_reward/mean": 0.35900117829442024, + "rewards/cross_solution_unittest_reward/std": 0.1959025040268898, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 62 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1132.0, + "completions/mean_length": 478.97137451171875, + "completions/mean_terminated_length": 470.8642272949219, + "completions/min_length": 156.0, + "completions/min_terminated_length": 156.0, + "entropy": 0.1028900146484375, + "epoch": 0.5748502994011976, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.034286729991436005, + "kl": 0.0025823116302490234, + "learning_rate": 8.014095333542548e-06, + "loss": 0.0114, + "num_tokens": 25945338.0, + "reward": 0.8456071615219116, + "reward_std": 0.2458418607711792, + "rewards/cross_solution_unittest_reward/mean": 0.37883561104536057, + "rewards/cross_solution_unittest_reward/std": 0.22371377050876617, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 63 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 855.0, + "completions/max_terminated_length": 855.0, + "completions/mean_length": 491.22137451171875, + "completions/mean_terminated_length": 491.22137451171875, + "completions/min_length": 309.0, + "completions/min_terminated_length": 309.0, + "entropy": 0.0994110107421875, + "epoch": 0.5839749073282008, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030411940068006516, + "kl": 0.0029740333557128906, + "learning_rate": 7.735032325742355e-06, + "loss": 0.0071, + "num_tokens": 26337343.0, + "reward": 1.2452019453048706, + "reward_std": 0.28348508477211, + "rewards/cross_solution_unittest_reward/mean": 0.5320437103509903, + "rewards/cross_solution_unittest_reward/std": 0.20597976446151733, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 64 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1100.0, + "completions/max_terminated_length": 1100.0, + "completions/mean_length": 481.04949951171875, + "completions/mean_terminated_length": 481.04949951171875, + "completions/min_length": 232.0, + "completions/min_terminated_length": 232.0, + "entropy": 0.1112823486328125, + "epoch": 0.5930995152552039, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03709479793906212, + "kl": 0.0034236907958984375, + "learning_rate": 7.4578166580651335e-06, + "loss": 0.0018, + "num_tokens": 26727642.0, + "reward": 1.0363634824752808, + "reward_std": 0.2708354592323303, + "rewards/cross_solution_unittest_reward/mean": 0.4516971707344055, + "rewards/cross_solution_unittest_reward/std": 0.30414242297410965, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 65 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.953125, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1223.0, + "completions/mean_length": 540.4427490234375, + "completions/mean_terminated_length": 516.4776611328125, + "completions/min_length": 225.0, + "completions/min_terminated_length": 225.0, + "entropy": 0.102630615234375, + "epoch": 0.602224123182207, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.031675394624471664, + "kl": 0.0034074783325195312, + "learning_rate": 7.182674431585703e-06, + "loss": -0.0053, + "num_tokens": 27207164.0, + "reward": 1.1464284658432007, + "reward_std": 0.23834940791130066, + "rewards/cross_solution_unittest_reward/mean": 0.4887780696153641, + "rewards/cross_solution_unittest_reward/std": 0.26997726410627365, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 66 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1062.0, + "completions/max_terminated_length": 1062.0, + "completions/mean_length": 492.33856201171875, + "completions/mean_terminated_length": 492.33856201171875, + "completions/min_length": 217.0, + "completions/min_terminated_length": 217.0, + "entropy": 0.098785400390625, + "epoch": 0.6113487311092102, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03432027995586395, + "kl": 0.004305839538574219, + "learning_rate": 6.909830056250527e-06, + "loss": 0.0091, + "num_tokens": 27661350.0, + "reward": 1.2568825483322144, + "reward_std": 0.3327004909515381, + "rewards/cross_solution_unittest_reward/mean": 0.5392695367336273, + "rewards/cross_solution_unittest_reward/std": 0.24271728843450546, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 67 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 953.0, + "completions/max_terminated_length": 953.0, + "completions/mean_length": 515.4479370117188, + "completions/mean_terminated_length": 515.4479370117188, + "completions/min_length": 270.0, + "completions/min_terminated_length": 270.0, + "entropy": 0.0980682373046875, + "epoch": 0.6204733390362133, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03175772726535797, + "kl": 0.0025773048400878906, + "learning_rate": 6.639506067845698e-06, + "loss": 0.0052, + "num_tokens": 28061562.0, + "reward": 1.0176376104354858, + "reward_std": 0.2250736653804779, + "rewards/cross_solution_unittest_reward/mean": 0.44822562485933304, + "rewards/cross_solution_unittest_reward/std": 0.2024165391921997, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 68 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 937.0, + "completions/max_terminated_length": 937.0, + "completions/mean_length": 493.3567810058594, + "completions/mean_terminated_length": 493.3567810058594, + "completions/min_length": 291.0, + "completions/min_terminated_length": 291.0, + "entropy": 0.0994415283203125, + "epoch": 0.6295979469632165, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033092062920331955, + "kl": 0.0027437210083007812, + "learning_rate": 6.3719229464935915e-06, + "loss": 0.0023, + "num_tokens": 28497043.0, + "reward": 1.0380425453186035, + "reward_std": 0.23625622689723969, + "rewards/cross_solution_unittest_reward/mean": 0.4579319655895233, + "rewards/cross_solution_unittest_reward/std": 0.20026205480098724, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 69 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1090.0, + "completions/max_terminated_length": 1090.0, + "completions/mean_length": 518.9921875, + "completions/mean_terminated_length": 518.9921875, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.0950927734375, + "epoch": 0.6387225548902196, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03034958802163601, + "kl": 0.0027964115142822266, + "learning_rate": 6.107298936826086e-06, + "loss": -0.001, + "num_tokens": 28942272.0, + "reward": 1.1260254383087158, + "reward_std": 0.3392567038536072, + "rewards/cross_solution_unittest_reward/mean": 0.4806702435016632, + "rewards/cross_solution_unittest_reward/std": 0.2181769162416458, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 70 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 983.0, + "completions/mean_length": 520.0963745117188, + "completions/mean_terminated_length": 504.05499267578125, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.1016082763671875, + "epoch": 0.6478471628172227, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03144148737192154, + "kl": 0.0028443336486816406, + "learning_rate": 5.845849869981137e-06, + "loss": 0.0139, + "num_tokens": 29354421.0, + "reward": 0.683957576751709, + "reward_std": 0.19793997704982758, + "rewards/cross_solution_unittest_reward/mean": 0.31807853654026985, + "rewards/cross_solution_unittest_reward/std": 0.18481089919805527, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 71 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1151.0, + "completions/max_terminated_length": 1151.0, + "completions/mean_length": 519.0833740234375, + "completions/mean_terminated_length": 519.0833740234375, + "completions/min_length": 273.0, + "completions/min_terminated_length": 273.0, + "entropy": 0.10321044921875, + "epoch": 0.6569717707442259, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.42725464701652527, + "kl": 0.03642606735229492, + "learning_rate": 5.587788987567785e-06, + "loss": 0.0002, + "num_tokens": 29764661.0, + "reward": 1.0907951593399048, + "reward_std": 0.25230711698532104, + "rewards/cross_solution_unittest_reward/mean": 0.47279611229896545, + "rewards/cross_solution_unittest_reward/std": 0.2695095092058182, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 72 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 791.0, + "completions/max_terminated_length": 791.0, + "completions/mean_length": 481.8125, + "completions/mean_terminated_length": 481.8125, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.1062469482421875, + "epoch": 0.666096378671229, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033676039427518845, + "kl": 0.003365039825439453, + "learning_rate": 5.333326767743263e-06, + "loss": 0.0049, + "num_tokens": 30167253.0, + "reward": 1.1704354286193848, + "reward_std": 0.26924002170562744, + "rewards/cross_solution_unittest_reward/mean": 0.503605455160141, + "rewards/cross_solution_unittest_reward/std": 0.24150928109884262, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 73 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1028.0, + "completions/max_terminated_length": 1028.0, + "completions/mean_length": 531.5989990234375, + "completions/mean_terminated_length": 531.5989990234375, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.098541259765625, + "epoch": 0.6752209865982322, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029249846935272217, + "kl": 0.0035860538482666016, + "learning_rate": 5.082670753543961e-06, + "loss": 0.0039, + "num_tokens": 30577827.0, + "reward": 1.0187840461730957, + "reward_std": 0.27179187536239624, + "rewards/cross_solution_unittest_reward/mean": 0.4446810930967331, + "rewards/cross_solution_unittest_reward/std": 0.2518850564956665, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 74 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1103.0, + "completions/max_terminated_length": 1103.0, + "completions/mean_length": 513.2943115234375, + "completions/mean_terminated_length": 513.2943115234375, + "completions/min_length": 238.0, + "completions/min_terminated_length": 238.0, + "entropy": 0.1102142333984375, + "epoch": 0.6843455945252352, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029439395293593407, + "kl": 0.002677440643310547, + "learning_rate": 4.836025383610382e-06, + "loss": 0.0061, + "num_tokens": 30956420.0, + "reward": 1.0420805215835571, + "reward_std": 0.2407897710800171, + "rewards/cross_solution_unittest_reward/mean": 0.4538075774908066, + "rewards/cross_solution_unittest_reward/std": 0.22261689603328705, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 75 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 958.0, + "completions/mean_length": 490.7734375, + "completions/mean_terminated_length": 474.57855224609375, + "completions/min_length": 233.0, + "completions/min_terminated_length": 233.0, + "entropy": 0.107452392578125, + "epoch": 0.6934702024522383, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.032771483063697815, + "kl": 0.0031430721282958984, + "learning_rate": 4.593591825444028e-06, + "loss": 0.0142, + "num_tokens": 31310765.0, + "reward": 0.8610422015190125, + "reward_std": 0.2242804616689682, + "rewards/cross_solution_unittest_reward/mean": 0.38095080852508545, + "rewards/cross_solution_unittest_reward/std": 0.2320489063858986, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 76 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 2310.0, + "completions/max_terminated_length": 2310.0, + "completions/mean_length": 502.9661560058594, + "completions/mean_terminated_length": 502.9661560058594, + "completions/min_length": 240.0, + "completions/min_terminated_length": 240.0, + "entropy": 0.10906982421875, + "epoch": 0.7025948103792415, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.036506183445453644, + "kl": 0.0029113292694091797, + "learning_rate": 4.355567811332311e-06, + "loss": -0.0079, + "num_tokens": 31718384.0, + "reward": 1.0806396007537842, + "reward_std": 0.2699912488460541, + "rewards/cross_solution_unittest_reward/mean": 0.4672403484582901, + "rewards/cross_solution_unittest_reward/std": 0.2118784487247467, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 77 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1254.0, + "completions/max_terminated_length": 1254.0, + "completions/mean_length": 517.2578125, + "completions/mean_terminated_length": 517.2578125, + "completions/min_length": 228.0, + "completions/min_terminated_length": 228.0, + "entropy": 0.09716796875, + "epoch": 0.7117194183062446, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.034147728234529495, + "kl": 0.0033721923828125, + "learning_rate": 4.12214747707527e-06, + "loss": 0.0056, + "num_tokens": 32162219.0, + "reward": 1.0363221168518066, + "reward_std": 0.3166886568069458, + "rewards/cross_solution_unittest_reward/mean": 0.4502222388982773, + "rewards/cross_solution_unittest_reward/std": 0.25721245259046555, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 78 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.953125, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 3357.0, + "completions/mean_length": 533.2734375, + "completions/mean_terminated_length": 509.251953125, + "completions/min_length": 263.0, + "completions/min_terminated_length": 263.0, + "entropy": 0.1002655029296875, + "epoch": 0.7208440262332478, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029844792559742928, + "kl": 0.0024819374084472656, + "learning_rate": 3.893521203645618e-06, + "loss": 0.0208, + "num_tokens": 32565772.0, + "reward": 1.0131210088729858, + "reward_std": 0.24623362720012665, + "rewards/cross_solution_unittest_reward/mean": 0.44952526688575745, + "rewards/cross_solution_unittest_reward/std": 0.233152836561203, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 79 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 822.0, + "completions/max_terminated_length": 822.0, + "completions/mean_length": 475.51824951171875, + "completions/mean_terminated_length": 475.51824951171875, + "completions/min_length": 223.0, + "completions/min_terminated_length": 223.0, + "entropy": 0.099822998046875, + "epoch": 0.7299686341602509, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03284022584557533, + "kl": 0.003690004348754883, + "learning_rate": 3.6698754619112974e-06, + "loss": -0.0014, + "num_tokens": 32926955.0, + "reward": 0.9502054452896118, + "reward_std": 0.23533646762371063, + "rewards/cross_solution_unittest_reward/mean": 0.43129874020814896, + "rewards/cross_solution_unittest_reward/std": 0.2330498918890953, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 80 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 962.0, + "completions/max_terminated_length": 962.0, + "completions/mean_length": 487.8177185058594, + "completions/mean_terminated_length": 487.8177185058594, + "completions/min_length": 200.0, + "completions/min_terminated_length": 200.0, + "entropy": 0.1024932861328125, + "epoch": 0.739093242087254, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.033087749034166336, + "kl": 0.002620697021484375, + "learning_rate": 3.4513926605471504e-06, + "loss": -0.0118, + "num_tokens": 33311189.0, + "reward": 1.0504975318908691, + "reward_std": 0.2200670838356018, + "rewards/cross_solution_unittest_reward/mean": 0.4615302234888077, + "rewards/cross_solution_unittest_reward/std": 0.20018253475427628, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 81 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1020.0, + "completions/max_terminated_length": 1020.0, + "completions/mean_length": 479.33074951171875, + "completions/mean_terminated_length": 479.33074951171875, + "completions/min_length": 203.0, + "completions/min_terminated_length": 203.0, + "entropy": 0.1038665771484375, + "epoch": 0.7482178500142572, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03461228311061859, + "kl": 0.002687692642211914, + "learning_rate": 3.2382509972598087e-06, + "loss": 0.0073, + "num_tokens": 33701588.0, + "reward": 0.9868416786193848, + "reward_std": 0.23717579245567322, + "rewards/cross_solution_unittest_reward/mean": 0.4351993724703789, + "rewards/cross_solution_unittest_reward/std": 0.2459121197462082, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 82 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1788.0, + "completions/max_terminated_length": 1788.0, + "completions/mean_length": 484.34375, + "completions/mean_terminated_length": 484.34375, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.1046905517578125, + "epoch": 0.7573424579412603, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.032532352954149246, + "kl": 0.0031991004943847656, + "learning_rate": 3.0306243134470668e-06, + "loss": 0.0136, + "num_tokens": 34096832.0, + "reward": 0.9118785858154297, + "reward_std": 0.27310025691986084, + "rewards/cross_solution_unittest_reward/mean": 0.39596662670373917, + "rewards/cross_solution_unittest_reward/std": 0.23184479027986526, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 83 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.953125, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 814.0, + "completions/mean_length": 497.4140625, + "completions/mean_terminated_length": 473.1102294921875, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.112152099609375, + "epoch": 0.7664670658682635, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030421627685427666, + "kl": 0.0025551319122314453, + "learning_rate": 2.8286819524103657e-06, + "loss": 0.015, + "num_tokens": 34443807.0, + "reward": 1.0494276285171509, + "reward_std": 0.24891619384288788, + "rewards/cross_solution_unittest_reward/mean": 0.4558604210615158, + "rewards/cross_solution_unittest_reward/std": 0.21075250953435898, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 84 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 963.0, + "completions/mean_length": 498.6770935058594, + "completions/mean_terminated_length": 490.6214294433594, + "completions/min_length": 267.0, + "completions/min_terminated_length": 267.0, + "entropy": 0.1007080078125, + "epoch": 0.7755916737952666, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030757782980799675, + "kl": 0.0036520957946777344, + "learning_rate": 2.6325886212359496e-06, + "loss": 0.0108, + "num_tokens": 34874923.0, + "reward": 1.0011277198791504, + "reward_std": 0.2030499428510666, + "rewards/cross_solution_unittest_reward/mean": 0.44105569273233414, + "rewards/cross_solution_unittest_reward/std": 0.19068162888288498, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 85 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1026.0, + "completions/max_terminated_length": 1026.0, + "completions/mean_length": 480.3359375, + "completions/mean_terminated_length": 480.3359375, + "completions/min_length": 243.0, + "completions/min_terminated_length": 243.0, + "entropy": 0.108489990234375, + "epoch": 0.7847162817222697, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.034772977232933044, + "kl": 0.0031616687774658203, + "learning_rate": 2.4425042564574186e-06, + "loss": -0.0031, + "num_tokens": 35288116.0, + "reward": 0.9796751737594604, + "reward_std": 0.23475846648216248, + "rewards/cross_solution_unittest_reward/mean": 0.43246912956237793, + "rewards/cross_solution_unittest_reward/std": 0.19666722416877747, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 86 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 3303.0, + "completions/mean_length": 531.0416870117188, + "completions/mean_terminated_length": 523.0704956054688, + "completions/min_length": 260.0, + "completions/min_terminated_length": 260.0, + "entropy": 0.10565185546875, + "epoch": 0.7938408896492729, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.039444971829652786, + "kl": 0.0033414363861083984, + "learning_rate": 2.2585838936091753e-06, + "loss": 0.0083, + "num_tokens": 35724444.0, + "reward": 1.1217626333236694, + "reward_std": 0.223866805434227, + "rewards/cross_solution_unittest_reward/mean": 0.48780302703380585, + "rewards/cross_solution_unittest_reward/std": 0.22542807459831238, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 87 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 878.0, + "completions/max_terminated_length": 878.0, + "completions/mean_length": 442.8828125, + "completions/mean_terminated_length": 442.8828125, + "completions/min_length": 235.0, + "completions/min_terminated_length": 235.0, + "entropy": 0.1145172119140625, + "epoch": 0.802965497576276, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03573368489742279, + "kl": 0.0030510425567626953, + "learning_rate": 2.0809775407772505e-06, + "loss": 0.0084, + "num_tokens": 36081911.0, + "reward": 1.0845204591751099, + "reward_std": 0.2791823744773865, + "rewards/cross_solution_unittest_reward/mean": 0.461356058716774, + "rewards/cross_solution_unittest_reward/std": 0.23068440705537796, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 88 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 852.0, + "completions/max_terminated_length": 852.0, + "completions/mean_length": 518.3568115234375, + "completions/mean_terminated_length": 518.3568115234375, + "completions/min_length": 313.0, + "completions/min_terminated_length": 313.0, + "entropy": 0.093414306640625, + "epoch": 0.8120901055032792, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03187531977891922, + "kl": 0.002918720245361328, + "learning_rate": 1.9098300562505266e-06, + "loss": 0.0067, + "num_tokens": 36510360.0, + "reward": 0.9905633330345154, + "reward_std": 0.20145916938781738, + "rewards/cross_solution_unittest_reward/mean": 0.4334751144051552, + "rewards/cross_solution_unittest_reward/std": 0.23444491624832153, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 89 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 986.0, + "completions/max_terminated_length": 986.0, + "completions/mean_length": 490.1953125, + "completions/mean_terminated_length": 490.1953125, + "completions/min_length": 258.0, + "completions/min_terminated_length": 258.0, + "entropy": 0.0989532470703125, + "epoch": 0.8212147134302823, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.05914483591914177, + "kl": 0.008898019790649414, + "learning_rate": 1.74528103037226e-06, + "loss": 0.0025, + "num_tokens": 36925595.0, + "reward": 0.9185196757316589, + "reward_std": 0.3170742094516754, + "rewards/cross_solution_unittest_reward/mean": 0.3952532112598419, + "rewards/cross_solution_unittest_reward/std": 0.2790588438510895, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 90 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 980.0, + "completions/max_terminated_length": 980.0, + "completions/mean_length": 483.3567810058594, + "completions/mean_terminated_length": 483.3567810058594, + "completions/min_length": 230.0, + "completions/min_terminated_length": 230.0, + "entropy": 0.10723876953125, + "epoch": 0.8303393213572854, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03479889780282974, + "kl": 0.0028510093688964844, + "learning_rate": 1.587464671688187e-06, + "loss": 0.001, + "num_tokens": 37317868.0, + "reward": 0.9646342992782593, + "reward_std": 0.18958084285259247, + "rewards/cross_solution_unittest_reward/mean": 0.4295145273208618, + "rewards/cross_solution_unittest_reward/std": 0.2108893096446991, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 91 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 956.0, + "completions/max_terminated_length": 956.0, + "completions/mean_length": 501.15887451171875, + "completions/mean_terminated_length": 501.15887451171875, + "completions/min_length": 237.0, + "completions/min_terminated_length": 237.0, + "entropy": 0.1046600341796875, + "epoch": 0.8394639292842886, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030385170131921768, + "kl": 0.0032634735107421875, + "learning_rate": 1.436509697484111e-06, + "loss": 0.0069, + "num_tokens": 37734865.0, + "reward": 0.9471985697746277, + "reward_std": 0.24106745421886444, + "rewards/cross_solution_unittest_reward/mean": 0.4128854274749756, + "rewards/cross_solution_unittest_reward/std": 0.24737022817134857, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 92 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1438.0, + "completions/mean_length": 564.9948120117188, + "completions/mean_terminated_length": 549.1884765625, + "completions/min_length": 280.0, + "completions/min_terminated_length": 280.0, + "entropy": 0.087799072265625, + "epoch": 0.8485885372112917, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.2799060642719269, + "kl": 0.017560720443725586, + "learning_rate": 1.2925392288022299e-06, + "loss": 0.0064, + "num_tokens": 38171079.0, + "reward": 0.9303452372550964, + "reward_std": 0.21974782645702362, + "rewards/cross_solution_unittest_reward/mean": 0.41144028306007385, + "rewards/cross_solution_unittest_reward/std": 0.23167048394680023, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 93 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.96875, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 1033.0, + "completions/mean_length": 545.6354370117188, + "completions/mean_terminated_length": 529.727783203125, + "completions/min_length": 204.0, + "completions/min_terminated_length": 204.0, + "entropy": 0.092437744140625, + "epoch": 0.8577131451382949, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02881380170583725, + "kl": 0.0027322769165039062, + "learning_rate": 1.1556706900218572e-06, + "loss": 0.0152, + "num_tokens": 38596603.0, + "reward": 0.9431265592575073, + "reward_std": 0.3126261830329895, + "rewards/cross_solution_unittest_reward/mean": 0.4096960350871086, + "rewards/cross_solution_unittest_reward/std": 0.274701751768589, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 94 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 775.0, + "completions/max_terminated_length": 775.0, + "completions/mean_length": 492.7734375, + "completions/mean_terminated_length": 492.7734375, + "completions/min_length": 255.0, + "completions/min_terminated_length": 255.0, + "entropy": 0.1008758544921875, + "epoch": 0.866837753065298, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.030817339196801186, + "kl": 0.003123760223388672, + "learning_rate": 1.0260157130864178e-06, + "loss": -0.0065, + "num_tokens": 39042508.0, + "reward": 0.98272705078125, + "reward_std": 0.2458946406841278, + "rewards/cross_solution_unittest_reward/mean": 0.42810578644275665, + "rewards/cross_solution_unittest_reward/std": 0.19119084626436234, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 95 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1011.0, + "completions/max_terminated_length": 1011.0, + "completions/mean_length": 520.4036865234375, + "completions/mean_terminated_length": 520.4036865234375, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.10015869140625, + "epoch": 0.8759623609923011, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03871850296854973, + "kl": 0.0033326148986816406, + "learning_rate": 9.036800464548157e-07, + "loss": -0.0045, + "num_tokens": 39430183.0, + "reward": 0.944609522819519, + "reward_std": 0.2243545949459076, + "rewards/cross_solution_unittest_reward/mean": 0.40971074998378754, + "rewards/cross_solution_unittest_reward/std": 0.22074688971042633, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 96 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 832.0, + "completions/mean_length": 457.82293701171875, + "completions/mean_terminated_length": 449.66058349609375, + "completions/min_length": 227.0, + "completions/min_terminated_length": 227.0, + "entropy": 0.1109771728515625, + "epoch": 0.8850869689193043, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.035750389099121094, + "kl": 0.0031118392944335938, + "learning_rate": 7.887634688515e-07, + "loss": -0.0005, + "num_tokens": 39751387.0, + "reward": 1.1470212936401367, + "reward_std": 0.19520321488380432, + "rewards/cross_solution_unittest_reward/mean": 0.5024754554033279, + "rewards/cross_solution_unittest_reward/std": 0.2238490730524063, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 97 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 856.0, + "completions/max_terminated_length": 856.0, + "completions/mean_length": 528.5989990234375, + "completions/mean_terminated_length": 528.5989990234375, + "completions/min_length": 285.0, + "completions/min_terminated_length": 285.0, + "entropy": 0.0987396240234375, + "epoch": 0.8942115768463074, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.02971605211496353, + "kl": 0.002849102020263672, + "learning_rate": 6.813597078854772e-07, + "loss": 0.0069, + "num_tokens": 40197641.0, + "reward": 1.0776381492614746, + "reward_std": 0.2849399149417877, + "rewards/cross_solution_unittest_reward/mean": 0.4694649428129196, + "rewards/cross_solution_unittest_reward/std": 0.28404664993286133, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 98 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 911.0, + "completions/max_terminated_length": 911.0, + "completions/mean_length": 493.4036560058594, + "completions/mean_terminated_length": 493.4036560058594, + "completions/min_length": 209.0, + "completions/min_terminated_length": 209.0, + "entropy": 0.1012420654296875, + "epoch": 0.9033361847733106, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03130873292684555, + "kl": 0.0025815963745117188, + "learning_rate": 5.815563636047539e-07, + "loss": -0.0035, + "num_tokens": 40569268.0, + "reward": 1.0923125743865967, + "reward_std": 0.24533653259277344, + "rewards/cross_solution_unittest_reward/mean": 0.4748411625623703, + "rewards/cross_solution_unittest_reward/std": 0.2197420746088028, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 99 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 914.0, + "completions/max_terminated_length": 914.0, + "completions/mean_length": 474.8125, + "completions/mean_terminated_length": 474.8125, + "completions/min_length": 261.0, + "completions/min_terminated_length": 261.0, + "entropy": 0.1113739013671875, + "epoch": 0.9124607927003137, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03394750878214836, + "kl": 0.003151416778564453, + "learning_rate": 4.894348370484648e-07, + "loss": 0.0001, + "num_tokens": 40993476.0, + "reward": 0.9635044932365417, + "reward_std": 0.24135762453079224, + "rewards/cross_solution_unittest_reward/mean": 0.4190831929445267, + "rewards/cross_solution_unittest_reward/std": 0.24994324147701263, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 100 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -4.984375, + "completions/max_length": 3584.0, + "completions/max_terminated_length": 848.0, + "completions/mean_length": 488.19012451171875, + "completions/mean_terminated_length": 480.1070556640625, + "completions/min_length": 248.0, + "completions/min_terminated_length": 248.0, + "entropy": 0.0947265625, + "epoch": 0.9215854006273168, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.034001559019088745, + "kl": 0.003081798553466797, + "learning_rate": 4.0507026385502747e-07, + "loss": 0.0004, + "num_tokens": 41383509.0, + "reward": 1.047755479812622, + "reward_std": 0.23132114112377167, + "rewards/cross_solution_unittest_reward/mean": 0.4617375209927559, + "rewards/cross_solution_unittest_reward/std": 0.20788954198360443, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 101 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 995.0, + "completions/max_terminated_length": 995.0, + "completions/mean_length": 502.48699951171875, + "completions/mean_terminated_length": 502.48699951171875, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.094207763671875, + "epoch": 0.93071000855432, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028330322355031967, + "kl": 0.002512693405151367, + "learning_rate": 3.2853145298042954e-07, + "loss": -0.0035, + "num_tokens": 41811784.0, + "reward": 1.1942728757858276, + "reward_std": 0.28333157300949097, + "rewards/cross_solution_unittest_reward/mean": 0.5126944035291672, + "rewards/cross_solution_unittest_reward/std": 0.1964806318283081, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 102 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 968.0, + "completions/max_terminated_length": 968.0, + "completions/mean_length": 471.36199951171875, + "completions/mean_terminated_length": 471.36199951171875, + "completions/min_length": 241.0, + "completions/min_terminated_length": 241.0, + "entropy": 0.103363037109375, + "epoch": 0.939834616481323, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0329383909702301, + "kl": 0.0033469200134277344, + "learning_rate": 2.5988083057666534e-07, + "loss": -0.0076, + "num_tokens": 42183707.0, + "reward": 1.1951045989990234, + "reward_std": 0.2977498471736908, + "rewards/cross_solution_unittest_reward/mean": 0.514451876282692, + "rewards/cross_solution_unittest_reward/std": 0.251402385532856, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 103 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 866.0, + "completions/max_terminated_length": 866.0, + "completions/mean_length": 500.29168701171875, + "completions/mean_terminated_length": 500.29168701171875, + "completions/min_length": 287.0, + "completions/min_terminated_length": 287.0, + "entropy": 0.09759521484375, + "epoch": 0.9489592244083263, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0342080220580101, + "kl": 0.0033740997314453125, + "learning_rate": 1.9917438907606556e-07, + "loss": 0.0017, + "num_tokens": 42602043.0, + "reward": 1.004610300064087, + "reward_std": 0.2067084014415741, + "rewards/cross_solution_unittest_reward/mean": 0.44767728447914124, + "rewards/cross_solution_unittest_reward/std": 0.19676169008016586, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 104 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 942.0, + "completions/max_terminated_length": 942.0, + "completions/mean_length": 461.2161560058594, + "completions/mean_terminated_length": 461.2161560058594, + "completions/min_length": 275.0, + "completions/min_terminated_length": 275.0, + "entropy": 0.1075592041015625, + "epoch": 0.9580838323353293, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.0386245995759964, + "kl": 0.0034677982330322266, + "learning_rate": 1.464616415230702e-07, + "loss": -0.0062, + "num_tokens": 42963262.0, + "reward": 0.8114880323410034, + "reward_std": 0.26597505807876587, + "rewards/cross_solution_unittest_reward/mean": 0.3618558570742607, + "rewards/cross_solution_unittest_reward/std": 0.23664288967847824, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 105 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 811.0, + "completions/max_terminated_length": 811.0, + "completions/mean_length": 524.5104370117188, + "completions/mean_terminated_length": 524.5104370117188, + "completions/min_length": 274.0, + "completions/min_terminated_length": 274.0, + "entropy": 0.093170166015625, + "epoch": 0.9672084402623324, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.028961867094039917, + "kl": 0.0027861595153808594, + "learning_rate": 1.0178558119067316e-07, + "loss": -0.0063, + "num_tokens": 43390042.0, + "reward": 1.000401496887207, + "reward_std": 0.30445563793182373, + "rewards/cross_solution_unittest_reward/mean": 0.43068091571331024, + "rewards/cross_solution_unittest_reward/std": 0.25178908556699753, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 106 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 885.0, + "completions/max_terminated_length": 885.0, + "completions/mean_length": 494.6015625, + "completions/mean_terminated_length": 494.6015625, + "completions/min_length": 282.0, + "completions/min_terminated_length": 282.0, + "entropy": 0.1114959716796875, + "epoch": 0.9763330481893356, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.034926630556583405, + "kl": 0.0031135082244873047, + "learning_rate": 6.51826465144978e-08, + "loss": -0.0033, + "num_tokens": 43794513.0, + "reward": 1.0083611011505127, + "reward_std": 0.23411409556865692, + "rewards/cross_solution_unittest_reward/mean": 0.4444109722971916, + "rewards/cross_solution_unittest_reward/std": 0.2296873927116394, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 107 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 1069.0, + "completions/max_terminated_length": 1069.0, + "completions/mean_length": 530.9869995117188, + "completions/mean_terminated_length": 530.9869995117188, + "completions/min_length": 252.0, + "completions/min_terminated_length": 252.0, + "entropy": 0.0896759033203125, + "epoch": 0.9854576561163387, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.029170462861657143, + "kl": 0.002541780471801758, + "learning_rate": 3.668269137308666e-08, + "loss": 0.0042, + "num_tokens": 44236428.0, + "reward": 1.2101051807403564, + "reward_std": 0.225676029920578, + "rewards/cross_solution_unittest_reward/mean": 0.525607168674469, + "rewards/cross_solution_unittest_reward/std": 0.20895672589540482, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 108 + }, + { + "clip_ratio/high_max": 0.0, + "clip_ratio/high_mean": 0.0, + "clip_ratio/low_mean": 0.0, + "clip_ratio/low_min": 0.0, + "clip_ratio/region_mean": 0.0, + "completions/clipped_ratio": -5.0, + "completions/max_length": 841.0, + "completions/max_terminated_length": 841.0, + "completions/mean_length": 468.3489685058594, + "completions/mean_terminated_length": 468.3489685058594, + "completions/min_length": 216.0, + "completions/min_terminated_length": 216.0, + "entropy": 0.101654052734375, + "epoch": 0.9945822640433419, + "frac_reward_zero_std": 0.0, + "grad_norm": 0.03142247721552849, + "kl": 0.002844095230102539, + "learning_rate": 1.630896073864352e-08, + "loss": 0.0032, + "num_tokens": 44609178.0, + "reward": 1.2266151905059814, + "reward_std": 0.24263408780097961, + "rewards/cross_solution_unittest_reward/mean": 0.5332132428884506, + "rewards/cross_solution_unittest_reward/std": 0.22311152517795563, + "rewards/format_reward/mean": 0.0, + "rewards/format_reward/std": 0.0, + "step": 109 + }, + { + "epoch": 0.9945822640433419, + "step": 109, + "total_flos": 0.0, + "train_loss": 0.0030027573193699003, + "train_runtime": 43116.833, + "train_samples_per_second": 0.03, + "train_steps_per_second": 0.003 + } + ], + "logging_steps": 1, + "max_steps": 110, + "num_input_tokens_seen": 44609178, + "num_train_epochs": 1, + "save_steps": 25, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": false + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 2, + "trial_name": null, + "trial_params": null +}