| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.9945822640433419, |
| "eval_steps": 500, |
| "global_step": 109, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1024.0, |
| "completions/max_terminated_length": 1024.0, |
| "completions/mean_length": 525.9375, |
| "completions/mean_terminated_length": 525.9375, |
| "completions/min_length": 263.0, |
| "completions/min_terminated_length": 263.0, |
| "entropy": 0.110321044921875, |
| "epoch": 0.009124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.025090789422392845, |
| "kl": 0.0002155303955078125, |
| "learning_rate": 2e-05, |
| "loss": 0.002, |
| "num_tokens": 410568.0, |
| "reward": 1.2271196842193604, |
| "reward_std": 0.31988272070884705, |
| "rewards/cross_solution_unittest_reward/mean": 0.47540418803691864, |
| "rewards/cross_solution_unittest_reward/std": 0.23665796220302582, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 1 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1937.0, |
| "completions/max_terminated_length": 1937.0, |
| "completions/mean_length": 493.22918701171875, |
| "completions/mean_terminated_length": 493.22918701171875, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.1114044189453125, |
| "epoch": 0.018249215854006275, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028324643149971962, |
| "kl": 0.00023484230041503906, |
| "learning_rate": 1.9995921928281893e-05, |
| "loss": -0.002, |
| "num_tokens": 821504.0, |
| "reward": 1.0042212009429932, |
| "reward_std": 0.2644613981246948, |
| "rewards/cross_solution_unittest_reward/mean": 0.39199119806289673, |
| "rewards/cross_solution_unittest_reward/std": 0.25940997898578644, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 2 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1133.0, |
| "completions/max_terminated_length": 1133.0, |
| "completions/mean_length": 567.2474365234375, |
| "completions/mean_terminated_length": 567.2474365234375, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.0968017578125, |
| "epoch": 0.02737382378100941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02467537671327591, |
| "kl": 0.00020578503608703613, |
| "learning_rate": 1.9983691039261358e-05, |
| "loss": 0.009, |
| "num_tokens": 1260095.0, |
| "reward": 1.0529532432556152, |
| "reward_std": 0.2820073366165161, |
| "rewards/cross_solution_unittest_reward/mean": 0.407609261572361, |
| "rewards/cross_solution_unittest_reward/std": 0.20918594300746918, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 3 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 779.0, |
| "completions/max_terminated_length": 779.0, |
| "completions/mean_length": 484.875, |
| "completions/mean_terminated_length": 484.875, |
| "completions/min_length": 258.0, |
| "completions/min_terminated_length": 258.0, |
| "entropy": 0.1236572265625, |
| "epoch": 0.03649843170801255, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04513426125049591, |
| "kl": 0.0002601146697998047, |
| "learning_rate": 1.9963317308626916e-05, |
| "loss": -0.0077, |
| "num_tokens": 1693503.0, |
| "reward": 0.7472698092460632, |
| "reward_std": 0.2714373469352722, |
| "rewards/cross_solution_unittest_reward/mean": 0.30792947113513947, |
| "rewards/cross_solution_unittest_reward/std": 0.1790020987391472, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 4 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2220.0, |
| "completions/max_terminated_length": 2220.0, |
| "completions/mean_length": 490.8828125, |
| "completions/mean_terminated_length": 490.8828125, |
| "completions/min_length": 242.0, |
| "completions/min_terminated_length": 242.0, |
| "entropy": 0.123046875, |
| "epoch": 0.04562303963501568, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029334085062146187, |
| "kl": 0.0002612471580505371, |
| "learning_rate": 1.99348173534855e-05, |
| "loss": -0.0056, |
| "num_tokens": 2076626.0, |
| "reward": 1.1481083631515503, |
| "reward_std": 0.3634147047996521, |
| "rewards/cross_solution_unittest_reward/mean": 0.44348418712615967, |
| "rewards/cross_solution_unittest_reward/std": 0.280821830034256, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 5 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1143.0, |
| "completions/mean_length": 545.15625, |
| "completions/mean_terminated_length": 529.24609375, |
| "completions/min_length": 270.0, |
| "completions/min_terminated_length": 270.0, |
| "entropy": 0.1153717041015625, |
| "epoch": 0.05474764756201882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027162108570337296, |
| "kl": 0.0002492964267730713, |
| "learning_rate": 1.989821441880933e-05, |
| "loss": -0.0044, |
| "num_tokens": 2500214.0, |
| "reward": 1.075005054473877, |
| "reward_std": 0.26096874475479126, |
| "rewards/cross_solution_unittest_reward/mean": 0.41980794072151184, |
| "rewards/cross_solution_unittest_reward/std": 0.24784592539072037, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 6 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1106.0, |
| "completions/mean_length": 534.25, |
| "completions/mean_terminated_length": 526.2872314453125, |
| "completions/min_length": 221.0, |
| "completions/min_terminated_length": 221.0, |
| "entropy": 0.1142120361328125, |
| "epoch": 0.06387225548902195, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.026346901431679726, |
| "kl": 0.00027573108673095703, |
| "learning_rate": 1.9853538358476933e-05, |
| "loss": -0.0033, |
| "num_tokens": 2945870.0, |
| "reward": 1.0205891132354736, |
| "reward_std": 0.3387899398803711, |
| "rewards/cross_solution_unittest_reward/mean": 0.4038037732243538, |
| "rewards/cross_solution_unittest_reward/std": 0.2187812626361847, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 7 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1321.0, |
| "completions/max_terminated_length": 1321.0, |
| "completions/mean_length": 525.65625, |
| "completions/mean_terminated_length": 525.65625, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.109466552734375, |
| "epoch": 0.0729968634160251, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027590544894337654, |
| "kl": 0.00027805566787719727, |
| "learning_rate": 1.9800825610923937e-05, |
| "loss": 0.0054, |
| "num_tokens": 3402378.0, |
| "reward": 1.002925157546997, |
| "reward_std": 0.2862505316734314, |
| "rewards/cross_solution_unittest_reward/mean": 0.3803572356700897, |
| "rewards/cross_solution_unittest_reward/std": 0.26964404433965683, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 8 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 950.0, |
| "completions/max_terminated_length": 950.0, |
| "completions/mean_length": 490.0833435058594, |
| "completions/mean_terminated_length": 490.0833435058594, |
| "completions/min_length": 210.0, |
| "completions/min_terminated_length": 210.0, |
| "entropy": 0.1181488037109375, |
| "epoch": 0.08212147134302823, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030266448855400085, |
| "kl": 0.00028058886528015137, |
| "learning_rate": 1.9740119169423337e-05, |
| "loss": 0.0072, |
| "num_tokens": 3794010.0, |
| "reward": 1.008456826210022, |
| "reward_std": 0.33921265602111816, |
| "rewards/cross_solution_unittest_reward/mean": 0.4010137543082237, |
| "rewards/cross_solution_unittest_reward/std": 0.22554735094308853, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 9 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1135.0, |
| "completions/max_terminated_length": 1135.0, |
| "completions/mean_length": 505.63543701171875, |
| "completions/mean_terminated_length": 505.63543701171875, |
| "completions/min_length": 259.0, |
| "completions/min_terminated_length": 259.0, |
| "entropy": 0.1077117919921875, |
| "epoch": 0.09124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028971431776881218, |
| "kl": 0.0002878904342651367, |
| "learning_rate": 1.9671468547019575e-05, |
| "loss": 0.004, |
| "num_tokens": 4238414.0, |
| "reward": 1.0463480949401855, |
| "reward_std": 0.30459851026535034, |
| "rewards/cross_solution_unittest_reward/mean": 0.40890535712242126, |
| "rewards/cross_solution_unittest_reward/std": 0.23788339644670486, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 10 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1150.0, |
| "completions/mean_length": 543.4557495117188, |
| "completions/mean_terminated_length": 535.5169677734375, |
| "completions/min_length": 270.0, |
| "completions/min_terminated_length": 270.0, |
| "entropy": 0.102996826171875, |
| "epoch": 0.1003706871970345, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027505753561854362, |
| "kl": 0.0003129541873931885, |
| "learning_rate": 1.9594929736144978e-05, |
| "loss": 0.0015, |
| "num_tokens": 4662925.0, |
| "reward": 1.0352768898010254, |
| "reward_std": 0.2669483423233032, |
| "rewards/cross_solution_unittest_reward/mean": 0.4138600826263428, |
| "rewards/cross_solution_unittest_reward/std": 0.20296787470579147, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 11 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1080.0, |
| "completions/mean_length": 506.15887451171875, |
| "completions/mean_terminated_length": 490.04449462890625, |
| "completions/min_length": 201.0, |
| "completions/min_terminated_length": 201.0, |
| "entropy": 0.1264190673828125, |
| "epoch": 0.10949529512403763, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031232472509145737, |
| "kl": 0.0004932880401611328, |
| "learning_rate": 1.9510565162951538e-05, |
| "loss": 0.0096, |
| "num_tokens": 5095338.0, |
| "reward": 0.8861899375915527, |
| "reward_std": 0.31088966131210327, |
| "rewards/cross_solution_unittest_reward/mean": 0.34614112228155136, |
| "rewards/cross_solution_unittest_reward/std": 0.22473014146089554, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 12 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1216.0, |
| "completions/max_terminated_length": 1216.0, |
| "completions/mean_length": 519.7396240234375, |
| "completions/mean_terminated_length": 519.7396240234375, |
| "completions/min_length": 222.0, |
| "completions/min_terminated_length": 222.0, |
| "entropy": 0.112884521484375, |
| "epoch": 0.11861990305104078, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0282323956489563, |
| "kl": 0.00036662817001342773, |
| "learning_rate": 1.941844363639525e-05, |
| "loss": -0.0078, |
| "num_tokens": 5502462.0, |
| "reward": 1.0762056112289429, |
| "reward_std": 0.25708380341529846, |
| "rewards/cross_solution_unittest_reward/mean": 0.4215054139494896, |
| "rewards/cross_solution_unittest_reward/std": 0.23641275614500046, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 13 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2663.0, |
| "completions/max_terminated_length": 2663.0, |
| "completions/mean_length": 620.8854370117188, |
| "completions/mean_terminated_length": 620.8854370117188, |
| "completions/min_length": 270.0, |
| "completions/min_terminated_length": 270.0, |
| "entropy": 0.1026458740234375, |
| "epoch": 0.1277445109780439, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.026703201234340668, |
| "kl": 0.00039651989936828613, |
| "learning_rate": 1.9318640292114526e-05, |
| "loss": 0.0004, |
| "num_tokens": 5972986.0, |
| "reward": 1.1370387077331543, |
| "reward_std": 0.24530315399169922, |
| "rewards/cross_solution_unittest_reward/mean": 0.4404096007347107, |
| "rewards/cross_solution_unittest_reward/std": 0.2456706538796425, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 14 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 934.0, |
| "completions/mean_length": 482.0364685058594, |
| "completions/mean_terminated_length": 457.6115417480469, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.126495361328125, |
| "epoch": 0.13686911890504705, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029596447944641113, |
| "kl": 0.0004467964172363281, |
| "learning_rate": 1.92112365311485e-05, |
| "loss": 0.0147, |
| "num_tokens": 6324176.0, |
| "reward": 1.1108325719833374, |
| "reward_std": 0.2962481379508972, |
| "rewards/cross_solution_unittest_reward/mean": 0.4290347397327423, |
| "rewards/cross_solution_unittest_reward/std": 0.24357028305530548, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 15 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 2795.0, |
| "completions/mean_length": 522.7630615234375, |
| "completions/mean_terminated_length": 514.770263671875, |
| "completions/min_length": 192.0, |
| "completions/min_terminated_length": 192.0, |
| "entropy": 0.1092681884765625, |
| "epoch": 0.1459937268320502, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028343815356492996, |
| "kl": 0.00048470497131347656, |
| "learning_rate": 1.9096319953545186e-05, |
| "loss": -0.0025, |
| "num_tokens": 6724061.0, |
| "reward": 1.055526614189148, |
| "reward_std": 0.34647518396377563, |
| "rewards/cross_solution_unittest_reward/mean": 0.41471436619758606, |
| "rewards/cross_solution_unittest_reward/std": 0.21236862987279892, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 16 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1336.0, |
| "completions/max_terminated_length": 1336.0, |
| "completions/mean_length": 508.3984375, |
| "completions/mean_terminated_length": 508.3984375, |
| "completions/min_length": 281.0, |
| "completions/min_terminated_length": 281.0, |
| "entropy": 0.1138763427734375, |
| "epoch": 0.15511833475905332, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028925545513629913, |
| "kl": 0.0004659891128540039, |
| "learning_rate": 1.8973984286913584e-05, |
| "loss": -0.0014, |
| "num_tokens": 7133206.0, |
| "reward": 1.2844151258468628, |
| "reward_std": 0.3014547526836395, |
| "rewards/cross_solution_unittest_reward/mean": 0.473222941160202, |
| "rewards/cross_solution_unittest_reward/std": 0.282610684633255, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 17 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1300.0, |
| "completions/mean_length": 531.9765625, |
| "completions/mean_terminated_length": 524.0078125, |
| "completions/min_length": 257.0, |
| "completions/min_terminated_length": 257.0, |
| "entropy": 0.108856201171875, |
| "epoch": 0.16424294268605646, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02900790423154831, |
| "kl": 0.0005418062210083008, |
| "learning_rate": 1.8844329309978146e-05, |
| "loss": 0.0016, |
| "num_tokens": 7567925.0, |
| "reward": 1.0913987159729004, |
| "reward_std": 0.34041401743888855, |
| "rewards/cross_solution_unittest_reward/mean": 0.41306161880493164, |
| "rewards/cross_solution_unittest_reward/std": 0.24276886880397797, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 18 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 906.0, |
| "completions/max_terminated_length": 906.0, |
| "completions/mean_length": 493.9817810058594, |
| "completions/mean_terminated_length": 493.9817810058594, |
| "completions/min_length": 241.0, |
| "completions/min_terminated_length": 241.0, |
| "entropy": 0.1160125732421875, |
| "epoch": 0.17336755061305958, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029141006991267204, |
| "kl": 0.000545203685760498, |
| "learning_rate": 1.8707460771197773e-05, |
| "loss": 0.0073, |
| "num_tokens": 7976102.0, |
| "reward": 1.0161412954330444, |
| "reward_std": 0.2866203784942627, |
| "rewards/cross_solution_unittest_reward/mean": 0.39064040780067444, |
| "rewards/cross_solution_unittest_reward/std": 0.2544485256075859, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 19 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1267.0, |
| "completions/max_terminated_length": 1267.0, |
| "completions/mean_length": 527.3229370117188, |
| "completions/mean_terminated_length": 527.3229370117188, |
| "completions/min_length": 290.0, |
| "completions/min_terminated_length": 290.0, |
| "entropy": 0.1096038818359375, |
| "epoch": 0.18249215854006273, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02903016284108162, |
| "kl": 0.0006221532821655273, |
| "learning_rate": 1.856349030251589e-05, |
| "loss": 0.0037, |
| "num_tokens": 8428962.0, |
| "reward": 1.0079458951950073, |
| "reward_std": 0.35072073340415955, |
| "rewards/cross_solution_unittest_reward/mean": 0.38429319858551025, |
| "rewards/cross_solution_unittest_reward/std": 0.22937388718128204, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 20 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1208.0, |
| "completions/max_terminated_length": 1208.0, |
| "completions/mean_length": 482.62762451171875, |
| "completions/mean_terminated_length": 482.62762451171875, |
| "completions/min_length": 198.0, |
| "completions/min_terminated_length": 198.0, |
| "entropy": 0.109039306640625, |
| "epoch": 0.19161676646706588, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02826613187789917, |
| "kl": 0.000597834587097168, |
| "learning_rate": 1.8412535328311813e-05, |
| "loss": 0.0041, |
| "num_tokens": 8812995.0, |
| "reward": 1.0695934295654297, |
| "reward_std": 0.25936269760131836, |
| "rewards/cross_solution_unittest_reward/mean": 0.42481929063796997, |
| "rewards/cross_solution_unittest_reward/std": 0.20526525378227234, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 21 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1185.0, |
| "completions/max_terminated_length": 1185.0, |
| "completions/mean_length": 505.578125, |
| "completions/mean_terminated_length": 505.578125, |
| "completions/min_length": 210.0, |
| "completions/min_terminated_length": 210.0, |
| "entropy": 0.11138916015625, |
| "epoch": 0.200741374394069, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03168891742825508, |
| "kl": 0.0010138154029846191, |
| "learning_rate": 1.825471896962774e-05, |
| "loss": -0.0122, |
| "num_tokens": 9188513.0, |
| "reward": 1.0664725303649902, |
| "reward_std": 0.3464021384716034, |
| "rewards/cross_solution_unittest_reward/mean": 0.40388014912605286, |
| "rewards/cross_solution_unittest_reward/std": 0.2562775984406471, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 22 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1357.0, |
| "completions/max_terminated_length": 1357.0, |
| "completions/mean_length": 507.8411560058594, |
| "completions/mean_terminated_length": 507.8411560058594, |
| "completions/min_length": 186.0, |
| "completions/min_terminated_length": 186.0, |
| "entropy": 0.1090545654296875, |
| "epoch": 0.20986598232107215, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030050111934542656, |
| "kl": 0.0007647275924682617, |
| "learning_rate": 1.8090169943749477e-05, |
| "loss": -0.0017, |
| "num_tokens": 9635068.0, |
| "reward": 1.2008159160614014, |
| "reward_std": 0.25730741024017334, |
| "rewards/cross_solution_unittest_reward/mean": 0.4557904154062271, |
| "rewards/cross_solution_unittest_reward/std": 0.22673149406909943, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 23 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 950.0, |
| "completions/max_terminated_length": 950.0, |
| "completions/mean_length": 500.6302185058594, |
| "completions/mean_terminated_length": 500.6302185058594, |
| "completions/min_length": 226.0, |
| "completions/min_terminated_length": 226.0, |
| "entropy": 0.1072540283203125, |
| "epoch": 0.21899059024807527, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027480732649564743, |
| "kl": 0.0007421970367431641, |
| "learning_rate": 1.7919022459222754e-05, |
| "loss": 0.0006, |
| "num_tokens": 10054982.0, |
| "reward": 0.9477907419204712, |
| "reward_std": 0.3180457353591919, |
| "rewards/cross_solution_unittest_reward/mean": 0.3783436641097069, |
| "rewards/cross_solution_unittest_reward/std": 0.25479088723659515, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 24 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1137.0, |
| "completions/max_terminated_length": 1137.0, |
| "completions/mean_length": 490.1536560058594, |
| "completions/mean_terminated_length": 490.1536560058594, |
| "completions/min_length": 240.0, |
| "completions/min_terminated_length": 240.0, |
| "entropy": 0.1211090087890625, |
| "epoch": 0.22811519817507842, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03646322712302208, |
| "kl": 0.0009087324142456055, |
| "learning_rate": 1.7741416106390828e-05, |
| "loss": 0.0011, |
| "num_tokens": 10456873.0, |
| "reward": 1.0844480991363525, |
| "reward_std": 0.2816399335861206, |
| "rewards/cross_solution_unittest_reward/mean": 0.4169381707906723, |
| "rewards/cross_solution_unittest_reward/std": 0.2162943333387375, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 25 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1184.0, |
| "completions/mean_length": 508.9739685058594, |
| "completions/mean_terminated_length": 500.9451904296875, |
| "completions/min_length": 225.0, |
| "completions/min_terminated_length": 225.0, |
| "entropy": 0.1131439208984375, |
| "epoch": 0.23723980610208156, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027677049860358238, |
| "kl": 0.0007776618003845215, |
| "learning_rate": 1.7557495743542586e-05, |
| "loss": -0.0092, |
| "num_tokens": 10819487.0, |
| "reward": 0.9719910621643066, |
| "reward_std": 0.28075867891311646, |
| "rewards/cross_solution_unittest_reward/mean": 0.3785587027668953, |
| "rewards/cross_solution_unittest_reward/std": 0.24748405069112778, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 26 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 725.0, |
| "completions/max_terminated_length": 725.0, |
| "completions/mean_length": 460.6380310058594, |
| "completions/mean_terminated_length": 460.6380310058594, |
| "completions/min_length": 209.0, |
| "completions/min_terminated_length": 209.0, |
| "entropy": 0.1148834228515625, |
| "epoch": 0.24636441402908468, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028393210843205452, |
| "kl": 0.0009175539016723633, |
| "learning_rate": 1.736741137876405e-05, |
| "loss": -0.0004, |
| "num_tokens": 11193260.0, |
| "reward": 1.155346155166626, |
| "reward_std": 0.2796667814254761, |
| "rewards/cross_solution_unittest_reward/mean": 0.45263613760471344, |
| "rewards/cross_solution_unittest_reward/std": 0.21176424622535706, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 27 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 966.0, |
| "completions/max_terminated_length": 966.0, |
| "completions/mean_length": 493.7942810058594, |
| "completions/mean_terminated_length": 493.7942810058594, |
| "completions/min_length": 251.0, |
| "completions/min_terminated_length": 251.0, |
| "entropy": 0.108154296875, |
| "epoch": 0.2554890219560878, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02867596223950386, |
| "kl": 0.001038193702697754, |
| "learning_rate": 1.7171318047589637e-05, |
| "loss": 0.0046, |
| "num_tokens": 11621069.0, |
| "reward": 1.267564296722412, |
| "reward_std": 0.28822484612464905, |
| "rewards/cross_solution_unittest_reward/mean": 0.4844939559698105, |
| "rewards/cross_solution_unittest_reward/std": 0.26973531395196915, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 28 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 998.0, |
| "completions/mean_length": 489.83074951171875, |
| "completions/mean_terminated_length": 481.751953125, |
| "completions/min_length": 281.0, |
| "completions/min_terminated_length": 281.0, |
| "entropy": 0.1177520751953125, |
| "epoch": 0.26461362988309095, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030956774950027466, |
| "kl": 0.0012102127075195312, |
| "learning_rate": 1.696937568655294e-05, |
| "loss": -0.0038, |
| "num_tokens": 12017156.0, |
| "reward": 1.182539701461792, |
| "reward_std": 0.231126606464386, |
| "rewards/cross_solution_unittest_reward/mean": 0.4465182274580002, |
| "rewards/cross_solution_unittest_reward/std": 0.28548523783683777, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 29 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 907.0, |
| "completions/mean_length": 530.5963745117188, |
| "completions/mean_terminated_length": 522.6240234375, |
| "completions/min_length": 225.0, |
| "completions/min_terminated_length": 225.0, |
| "entropy": 0.1051483154296875, |
| "epoch": 0.2737382378100941, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030249865725636482, |
| "kl": 0.0010318756103515625, |
| "learning_rate": 1.6761749002740195e-05, |
| "loss": -0.0066, |
| "num_tokens": 12439785.0, |
| "reward": 0.9416704177856445, |
| "reward_std": 0.31163209676742554, |
| "rewards/cross_solution_unittest_reward/mean": 0.37286702543497086, |
| "rewards/cross_solution_unittest_reward/std": 0.26768606156110764, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 30 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 832.0, |
| "completions/max_terminated_length": 832.0, |
| "completions/mean_length": 461.34637451171875, |
| "completions/mean_terminated_length": 461.34637451171875, |
| "completions/min_length": 214.0, |
| "completions/min_terminated_length": 214.0, |
| "entropy": 0.11480712890625, |
| "epoch": 0.28286284573709725, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03138723969459534, |
| "kl": 0.0012290477752685547, |
| "learning_rate": 1.6548607339452853e-05, |
| "loss": 0.0012, |
| "num_tokens": 12800374.0, |
| "reward": 1.142934799194336, |
| "reward_std": 0.2438894510269165, |
| "rewards/cross_solution_unittest_reward/mean": 0.4574270248413086, |
| "rewards/cross_solution_unittest_reward/std": 0.20738767087459564, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 31 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 954.0, |
| "completions/max_terminated_length": 954.0, |
| "completions/mean_length": 484.15106201171875, |
| "completions/mean_terminated_length": 484.15106201171875, |
| "completions/min_length": 266.0, |
| "completions/min_terminated_length": 266.0, |
| "entropy": 0.10296630859375, |
| "epoch": 0.2919874536641004, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.037726812064647675, |
| "kl": 0.0012822151184082031, |
| "learning_rate": 1.6330124538088705e-05, |
| "loss": 0.0074, |
| "num_tokens": 13254672.0, |
| "reward": 1.2426599264144897, |
| "reward_std": 0.39957040548324585, |
| "rewards/cross_solution_unittest_reward/mean": 0.4713882952928543, |
| "rewards/cross_solution_unittest_reward/std": 0.23567520081996918, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 32 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 997.0, |
| "completions/max_terminated_length": 997.0, |
| "completions/mean_length": 474.6171875, |
| "completions/mean_terminated_length": 474.6171875, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.1254119873046875, |
| "epoch": 0.3011120615911035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03023543767631054, |
| "kl": 0.0014002323150634766, |
| "learning_rate": 1.6106478796354382e-05, |
| "loss": -0.0059, |
| "num_tokens": 13634901.0, |
| "reward": 1.0120811462402344, |
| "reward_std": 0.24670013785362244, |
| "rewards/cross_solution_unittest_reward/mean": 0.4261118918657303, |
| "rewards/cross_solution_unittest_reward/std": 0.2368394210934639, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 33 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 880.0, |
| "completions/max_terminated_length": 880.0, |
| "completions/mean_length": 492.13543701171875, |
| "completions/mean_terminated_length": 492.13543701171875, |
| "completions/min_length": 231.0, |
| "completions/min_terminated_length": 231.0, |
| "entropy": 0.1062469482421875, |
| "epoch": 0.31023666951810663, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030524957925081253, |
| "kl": 0.0013687610626220703, |
| "learning_rate": 1.5877852522924733e-05, |
| "loss": -0.006, |
| "num_tokens": 14063145.0, |
| "reward": 1.219003438949585, |
| "reward_std": 0.2584400475025177, |
| "rewards/cross_solution_unittest_reward/mean": 0.4796546399593353, |
| "rewards/cross_solution_unittest_reward/std": 0.23066769540309906, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 34 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 885.0, |
| "completions/max_terminated_length": 885.0, |
| "completions/mean_length": 498.69012451171875, |
| "completions/mean_terminated_length": 498.69012451171875, |
| "completions/min_length": 274.0, |
| "completions/min_terminated_length": 274.0, |
| "entropy": 0.1126251220703125, |
| "epoch": 0.3193612774451098, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030774278566241264, |
| "kl": 0.0014193058013916016, |
| "learning_rate": 1.5644432188667695e-05, |
| "loss": 0.0139, |
| "num_tokens": 14445962.0, |
| "reward": 1.1249631643295288, |
| "reward_std": 0.32856079936027527, |
| "rewards/cross_solution_unittest_reward/mean": 0.4171721339225769, |
| "rewards/cross_solution_unittest_reward/std": 0.24577928334474564, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 35 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1494.0, |
| "completions/max_terminated_length": 1494.0, |
| "completions/mean_length": 512.0911865234375, |
| "completions/mean_terminated_length": 512.0911865234375, |
| "completions/min_length": 276.0, |
| "completions/min_terminated_length": 276.0, |
| "entropy": 0.1166534423828125, |
| "epoch": 0.32848588537211293, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03613327443599701, |
| "kl": 0.002481698989868164, |
| "learning_rate": 1.5406408174555978e-05, |
| "loss": 0.0038, |
| "num_tokens": 14864613.0, |
| "reward": 1.0395325422286987, |
| "reward_std": 0.31884095072746277, |
| "rewards/cross_solution_unittest_reward/mean": 0.3926886022090912, |
| "rewards/cross_solution_unittest_reward/std": 0.22794856131076813, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 36 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1651.0, |
| "completions/max_terminated_length": 1651.0, |
| "completions/mean_length": 516.1224365234375, |
| "completions/mean_terminated_length": 516.1224365234375, |
| "completions/min_length": 236.0, |
| "completions/min_terminated_length": 236.0, |
| "entropy": 0.0965118408203125, |
| "epoch": 0.3376104932991161, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029742714017629623, |
| "kl": 0.0016620159149169922, |
| "learning_rate": 1.5163974616389621e-05, |
| "loss": 0.0001, |
| "num_tokens": 15273692.0, |
| "reward": 1.0600379705429077, |
| "reward_std": 0.26911425590515137, |
| "rewards/cross_solution_unittest_reward/mean": 0.42146605998277664, |
| "rewards/cross_solution_unittest_reward/std": 0.23059702664613724, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 37 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 818.0, |
| "completions/mean_length": 496.34637451171875, |
| "completions/mean_terminated_length": 480.1806335449219, |
| "completions/min_length": 193.0, |
| "completions/min_terminated_length": 193.0, |
| "entropy": 0.110321044921875, |
| "epoch": 0.34673510122611917, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030567364767193794, |
| "kl": 0.0016117095947265625, |
| "learning_rate": 1.491732924645604e-05, |
| "loss": 0.0136, |
| "num_tokens": 15662473.0, |
| "reward": 1.1122747659683228, |
| "reward_std": 0.26910364627838135, |
| "rewards/cross_solution_unittest_reward/mean": 0.4311128109693527, |
| "rewards/cross_solution_unittest_reward/std": 0.23016689717769623, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 38 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 962.0, |
| "completions/max_terminated_length": 962.0, |
| "completions/mean_length": 473.0859375, |
| "completions/mean_terminated_length": 473.0859375, |
| "completions/min_length": 162.0, |
| "completions/min_terminated_length": 162.0, |
| "entropy": 0.117919921875, |
| "epoch": 0.3558597091531223, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03569599241018295, |
| "kl": 0.002121448516845703, |
| "learning_rate": 1.4666673232256738e-05, |
| "loss": 0.0213, |
| "num_tokens": 15974274.0, |
| "reward": 0.9712950587272644, |
| "reward_std": 0.37336552143096924, |
| "rewards/cross_solution_unittest_reward/mean": 0.37921562045812607, |
| "rewards/cross_solution_unittest_reward/std": 0.21440216898918152, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 39 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1488.0, |
| "completions/max_terminated_length": 1488.0, |
| "completions/mean_length": 496.86981201171875, |
| "completions/mean_terminated_length": 496.86981201171875, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.104644775390625, |
| "epoch": 0.36498431708012546, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03822171315550804, |
| "kl": 0.0029592514038085938, |
| "learning_rate": 1.4412211012432213e-05, |
| "loss": -0.0099, |
| "num_tokens": 16385736.0, |
| "reward": 0.9362239241600037, |
| "reward_std": 0.23804768919944763, |
| "rewards/cross_solution_unittest_reward/mean": 0.3656328096985817, |
| "rewards/cross_solution_unittest_reward/std": 0.28275594860315323, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 40 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 2157.0, |
| "completions/max_terminated_length": 2157.0, |
| "completions/mean_length": 504.46356201171875, |
| "completions/mean_terminated_length": 504.46356201171875, |
| "completions/min_length": 250.0, |
| "completions/min_terminated_length": 250.0, |
| "entropy": 0.1143646240234375, |
| "epoch": 0.3741089250071286, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032618340104818344, |
| "kl": 0.0024976730346679688, |
| "learning_rate": 1.4154150130018867e-05, |
| "loss": -0.0, |
| "num_tokens": 16799306.0, |
| "reward": 0.8250623941421509, |
| "reward_std": 0.2232421487569809, |
| "rewards/cross_solution_unittest_reward/mean": 0.3419858068227768, |
| "rewards/cross_solution_unittest_reward/std": 0.2102661430835724, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 41 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 875.0, |
| "completions/max_terminated_length": 875.0, |
| "completions/mean_length": 505.9583435058594, |
| "completions/mean_terminated_length": 505.9583435058594, |
| "completions/min_length": 275.0, |
| "completions/min_terminated_length": 275.0, |
| "entropy": 0.095428466796875, |
| "epoch": 0.38323353293413176, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03039792738854885, |
| "kl": 0.0020034313201904297, |
| "learning_rate": 1.3892701063173917e-05, |
| "loss": 0.0116, |
| "num_tokens": 17244018.0, |
| "reward": 1.0598524808883667, |
| "reward_std": 0.22255584597587585, |
| "rewards/cross_solution_unittest_reward/mean": 0.4190962612628937, |
| "rewards/cross_solution_unittest_reward/std": 0.24228056520223618, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 42 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 744.0, |
| "completions/mean_length": 487.9270935058594, |
| "completions/mean_terminated_length": 479.8433532714844, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.1062774658203125, |
| "epoch": 0.39235814086113485, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03161928430199623, |
| "kl": 0.0024061203002929688, |
| "learning_rate": 1.362807705350641e-05, |
| "loss": -0.0093, |
| "num_tokens": 17585734.0, |
| "reward": 1.223128318786621, |
| "reward_std": 0.23245906829833984, |
| "rewards/cross_solution_unittest_reward/mean": 0.47970961034297943, |
| "rewards/cross_solution_unittest_reward/std": 0.24992112070322037, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 43 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1550.0, |
| "completions/mean_length": 522.5390625, |
| "completions/mean_terminated_length": 506.5104675292969, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.0976715087890625, |
| "epoch": 0.401482748788138, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029316462576389313, |
| "kl": 0.0023124217987060547, |
| "learning_rate": 1.3360493932154301e-05, |
| "loss": 0.0087, |
| "num_tokens": 17997293.0, |
| "reward": 0.770310640335083, |
| "reward_std": 0.22640717029571533, |
| "rewards/cross_solution_unittest_reward/mean": 0.3347120136022568, |
| "rewards/cross_solution_unittest_reward/std": 0.2088947519659996, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 44 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 761.0, |
| "completions/max_terminated_length": 761.0, |
| "completions/mean_length": 452.7578125, |
| "completions/mean_terminated_length": 452.7578125, |
| "completions/min_length": 201.0, |
| "completions/min_terminated_length": 201.0, |
| "entropy": 0.117156982421875, |
| "epoch": 0.41060735671514115, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03837798163294792, |
| "kl": 0.003719329833984375, |
| "learning_rate": 1.3090169943749475e-05, |
| "loss": 0.008, |
| "num_tokens": 18373704.0, |
| "reward": 1.1445835828781128, |
| "reward_std": 0.27869975566864014, |
| "rewards/cross_solution_unittest_reward/mean": 0.4463556706905365, |
| "rewards/cross_solution_unittest_reward/std": 0.2553638890385628, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 45 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 950.0, |
| "completions/max_terminated_length": 950.0, |
| "completions/mean_length": 496.21875, |
| "completions/mean_terminated_length": 496.21875, |
| "completions/min_length": 258.0, |
| "completions/min_terminated_length": 258.0, |
| "entropy": 0.10125732421875, |
| "epoch": 0.4197319646421443, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032201867550611496, |
| "kl": 0.002789020538330078, |
| "learning_rate": 1.2817325568414299e-05, |
| "loss": 0.0053, |
| "num_tokens": 18784484.0, |
| "reward": 1.0264325141906738, |
| "reward_std": 0.3673638701438904, |
| "rewards/cross_solution_unittest_reward/mean": 0.40224773436784744, |
| "rewards/cross_solution_unittest_reward/std": 0.2333994135260582, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 46 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1553.0, |
| "completions/mean_length": 507.47137451171875, |
| "completions/mean_terminated_length": 491.3638916015625, |
| "completions/min_length": 252.0, |
| "completions/min_terminated_length": 252.0, |
| "entropy": 0.1202850341796875, |
| "epoch": 0.42885657256914744, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032994262874126434, |
| "kl": 0.0027620792388916016, |
| "learning_rate": 1.2542183341934873e-05, |
| "loss": 0.0142, |
| "num_tokens": 19204529.0, |
| "reward": 0.8056275844573975, |
| "reward_std": 0.2351686954498291, |
| "rewards/cross_solution_unittest_reward/mean": 0.32472602277994156, |
| "rewards/cross_solution_unittest_reward/std": 0.2546079531311989, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 47 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 893.0, |
| "completions/max_terminated_length": 893.0, |
| "completions/mean_length": 490.56512451171875, |
| "completions/mean_terminated_length": 490.56512451171875, |
| "completions/min_length": 264.0, |
| "completions/min_terminated_length": 264.0, |
| "entropy": 0.107147216796875, |
| "epoch": 0.43798118049615054, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031603384763002396, |
| "kl": 0.0028133392333984375, |
| "learning_rate": 1.2264967674257647e-05, |
| "loss": 0.0043, |
| "num_tokens": 19594874.0, |
| "reward": 1.1109874248504639, |
| "reward_std": 0.3147701919078827, |
| "rewards/cross_solution_unittest_reward/mean": 0.4285571426153183, |
| "rewards/cross_solution_unittest_reward/std": 0.2294345498085022, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 48 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1107.0, |
| "completions/mean_length": 502.71356201171875, |
| "completions/mean_terminated_length": 494.6684265136719, |
| "completions/min_length": 267.0, |
| "completions/min_terminated_length": 267.0, |
| "entropy": 0.100433349609375, |
| "epoch": 0.4471057884231537, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0326952300965786, |
| "kl": 0.0027174949645996094, |
| "learning_rate": 1.1985904666457455e-05, |
| "loss": 0.01, |
| "num_tokens": 20009764.0, |
| "reward": 0.8557624816894531, |
| "reward_std": 0.3132660984992981, |
| "rewards/cross_solution_unittest_reward/mean": 0.32815027981996536, |
| "rewards/cross_solution_unittest_reward/std": 0.2442621961236, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 49 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1427.0, |
| "completions/max_terminated_length": 1427.0, |
| "completions/mean_length": 507.59375, |
| "completions/mean_terminated_length": 507.59375, |
| "completions/min_length": 296.0, |
| "completions/min_terminated_length": 296.0, |
| "entropy": 0.0999755859375, |
| "epoch": 0.45623039635015683, |
| "frac_reward_zero_std": 0.0833333358168602, |
| "grad_norm": 0.03101285733282566, |
| "kl": 0.004076242446899414, |
| "learning_rate": 1.170522192632624e-05, |
| "loss": -0.004, |
| "num_tokens": 20420760.0, |
| "reward": 1.0653202533721924, |
| "reward_std": 0.3440527319908142, |
| "rewards/cross_solution_unittest_reward/mean": 0.41321125626564026, |
| "rewards/cross_solution_unittest_reward/std": 0.28402765840291977, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 50 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1267.0, |
| "completions/max_terminated_length": 1267.0, |
| "completions/mean_length": 507.9349060058594, |
| "completions/mean_terminated_length": 507.9349060058594, |
| "completions/min_length": 182.0, |
| "completions/min_terminated_length": 182.0, |
| "entropy": 0.098052978515625, |
| "epoch": 0.46535500427716, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03185436502099037, |
| "kl": 0.003798246383666992, |
| "learning_rate": 1.1423148382732854e-05, |
| "loss": 0.0012, |
| "num_tokens": 20840495.0, |
| "reward": 1.1997592449188232, |
| "reward_std": 0.2847609221935272, |
| "rewards/cross_solution_unittest_reward/mean": 0.4686891883611679, |
| "rewards/cross_solution_unittest_reward/std": 0.19803649932146072, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 51 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 920.0, |
| "completions/mean_length": 490.5989685058594, |
| "completions/mean_terminated_length": 482.5221862792969, |
| "completions/min_length": 215.0, |
| "completions/min_terminated_length": 215.0, |
| "entropy": 0.1027984619140625, |
| "epoch": 0.4744796122041631, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04125326871871948, |
| "kl": 0.004448890686035156, |
| "learning_rate": 1.1139914098905406e-05, |
| "loss": 0.0064, |
| "num_tokens": 21235525.0, |
| "reward": 1.196807861328125, |
| "reward_std": 0.3041377067565918, |
| "rewards/cross_solution_unittest_reward/mean": 0.4725197106599808, |
| "rewards/cross_solution_unittest_reward/std": 0.2374914139509201, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 52 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 836.0, |
| "completions/max_terminated_length": 836.0, |
| "completions/mean_length": 490.2421875, |
| "completions/mean_terminated_length": 490.2421875, |
| "completions/min_length": 257.0, |
| "completions/min_terminated_length": 257.0, |
| "entropy": 0.0958709716796875, |
| "epoch": 0.4836042201311662, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0320790559053421, |
| "kl": 0.002828359603881836, |
| "learning_rate": 1.08557500847884e-05, |
| "loss": 0.0042, |
| "num_tokens": 21637346.0, |
| "reward": 0.9321694374084473, |
| "reward_std": 0.2492300122976303, |
| "rewards/cross_solution_unittest_reward/mean": 0.3783368095755577, |
| "rewards/cross_solution_unittest_reward/std": 0.2201835662126541, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 53 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 994.0, |
| "completions/max_terminated_length": 994.0, |
| "completions/mean_length": 538.9453125, |
| "completions/mean_terminated_length": 538.9453125, |
| "completions/min_length": 291.0, |
| "completions/min_terminated_length": 291.0, |
| "entropy": 0.103179931640625, |
| "epoch": 0.49272882805816937, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03167517110705376, |
| "kl": 0.0037932395935058594, |
| "learning_rate": 1.0570888108627682e-05, |
| "loss": 0.0041, |
| "num_tokens": 22100661.0, |
| "reward": 0.9747004508972168, |
| "reward_std": 0.33161452412605286, |
| "rewards/cross_solution_unittest_reward/mean": 0.37912487238645554, |
| "rewards/cross_solution_unittest_reward/std": 0.20974859595298767, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 54 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 895.0, |
| "completions/max_terminated_length": 895.0, |
| "completions/mean_length": 479.8125, |
| "completions/mean_terminated_length": 479.8125, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.106231689453125, |
| "epoch": 0.5018534359851725, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03148855268955231, |
| "kl": 0.003476381301879883, |
| "learning_rate": 1.0285560507936962e-05, |
| "loss": 0.0042, |
| "num_tokens": 22473749.0, |
| "reward": 0.9761655330657959, |
| "reward_std": 0.2898656129837036, |
| "rewards/cross_solution_unittest_reward/mean": 0.37648654729127884, |
| "rewards/cross_solution_unittest_reward/std": 0.21749861538410187, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 55 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 952.0, |
| "completions/max_terminated_length": 952.0, |
| "completions/mean_length": 487.0442810058594, |
| "completions/mean_terminated_length": 487.0442810058594, |
| "completions/min_length": 230.0, |
| "completions/min_terminated_length": 230.0, |
| "entropy": 0.1035308837890625, |
| "epoch": 0.5109780439121756, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05067846551537514, |
| "kl": 0.005573272705078125, |
| "learning_rate": 1e-05, |
| "loss": 0.0098, |
| "num_tokens": 22845742.0, |
| "reward": 1.0181424617767334, |
| "reward_std": 0.238634392619133, |
| "rewards/cross_solution_unittest_reward/mean": 0.4077446535229683, |
| "rewards/cross_solution_unittest_reward/std": 0.21635101735591888, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 56 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 824.0, |
| "completions/mean_length": 498.7552185058594, |
| "completions/mean_terminated_length": 490.6997375488281, |
| "completions/min_length": 300.0, |
| "completions/min_terminated_length": 300.0, |
| "entropy": 0.104248046875, |
| "epoch": 0.5201026518391788, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033629678189754486, |
| "kl": 0.0038862228393554688, |
| "learning_rate": 9.71443949206304e-06, |
| "loss": 0.0042, |
| "num_tokens": 23289888.0, |
| "reward": 1.072588324546814, |
| "reward_std": 0.23051133751869202, |
| "rewards/cross_solution_unittest_reward/mean": 0.4333711564540863, |
| "rewards/cross_solution_unittest_reward/std": 0.21058296412229538, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 57 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1087.0, |
| "completions/mean_length": 494.6536560058594, |
| "completions/mean_terminated_length": 478.47906494140625, |
| "completions/min_length": 237.0, |
| "completions/min_terminated_length": 237.0, |
| "entropy": 0.1170196533203125, |
| "epoch": 0.5292272597661819, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03605455532670021, |
| "kl": 0.004413604736328125, |
| "learning_rate": 9.42911189137232e-06, |
| "loss": -0.0017, |
| "num_tokens": 23705051.0, |
| "reward": 1.092246413230896, |
| "reward_std": 0.2363906353712082, |
| "rewards/cross_solution_unittest_reward/mean": 0.42197316884994507, |
| "rewards/cross_solution_unittest_reward/std": 0.2724035382270813, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 58 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 972.0, |
| "completions/mean_length": 477.6224060058594, |
| "completions/mean_terminated_length": 469.5117492675781, |
| "completions/min_length": 226.0, |
| "completions/min_terminated_length": 226.0, |
| "entropy": 0.11468505859375, |
| "epoch": 0.5383518676931851, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.035716503858566284, |
| "kl": 0.004600048065185547, |
| "learning_rate": 9.144249915211605e-06, |
| "loss": 0.0055, |
| "num_tokens": 24109890.0, |
| "reward": 1.119308352470398, |
| "reward_std": 0.28101611137390137, |
| "rewards/cross_solution_unittest_reward/mean": 0.4181726723909378, |
| "rewards/cross_solution_unittest_reward/std": 0.25985223799943924, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 59 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 785.0, |
| "completions/max_terminated_length": 785.0, |
| "completions/mean_length": 492.97918701171875, |
| "completions/mean_terminated_length": 492.97918701171875, |
| "completions/min_length": 286.0, |
| "completions/min_terminated_length": 286.0, |
| "entropy": 0.09771728515625, |
| "epoch": 0.5474764756201882, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0333021804690361, |
| "kl": 0.004182338714599609, |
| "learning_rate": 8.860085901094595e-06, |
| "loss": -0.0062, |
| "num_tokens": 24543226.0, |
| "reward": 1.0735642910003662, |
| "reward_std": 0.347337543964386, |
| "rewards/cross_solution_unittest_reward/mean": 0.4236568510532379, |
| "rewards/cross_solution_unittest_reward/std": 0.22879399359226227, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 60 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 943.0, |
| "completions/max_terminated_length": 943.0, |
| "completions/mean_length": 529.3385620117188, |
| "completions/mean_terminated_length": 529.3385620117188, |
| "completions/min_length": 288.0, |
| "completions/min_terminated_length": 288.0, |
| "entropy": 0.097503662109375, |
| "epoch": 0.5566010835471913, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03290291503071785, |
| "kl": 0.003975391387939453, |
| "learning_rate": 8.576851617267151e-06, |
| "loss": -0.0023, |
| "num_tokens": 24991796.0, |
| "reward": 1.1211392879486084, |
| "reward_std": 0.28648895025253296, |
| "rewards/cross_solution_unittest_reward/mean": 0.4426130950450897, |
| "rewards/cross_solution_unittest_reward/std": 0.26173052191734314, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 61 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1262.0, |
| "completions/mean_length": 533.1823120117188, |
| "completions/mean_terminated_length": 525.2167358398438, |
| "completions/min_length": 284.0, |
| "completions/min_terminated_length": 284.0, |
| "entropy": 0.1041717529296875, |
| "epoch": 0.5657256914741945, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032658372074365616, |
| "kl": 0.0042781829833984375, |
| "learning_rate": 8.294778073673762e-06, |
| "loss": 0.0154, |
| "num_tokens": 25421578.0, |
| "reward": 0.8346015810966492, |
| "reward_std": 0.2451450526714325, |
| "rewards/cross_solution_unittest_reward/mean": 0.3612000532448292, |
| "rewards/cross_solution_unittest_reward/std": 0.18463484197854996, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 62 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 943.0, |
| "completions/max_terminated_length": 943.0, |
| "completions/mean_length": 465.7630310058594, |
| "completions/mean_terminated_length": 465.7630310058594, |
| "completions/min_length": 271.0, |
| "completions/min_terminated_length": 271.0, |
| "entropy": 0.1036376953125, |
| "epoch": 0.5748502994011976, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05787957087159157, |
| "kl": 0.005153656005859375, |
| "learning_rate": 8.014095333542548e-06, |
| "loss": 0.0039, |
| "num_tokens": 25789535.0, |
| "reward": 0.9333815574645996, |
| "reward_std": 0.259033203125, |
| "rewards/cross_solution_unittest_reward/mean": 0.37771864980459213, |
| "rewards/cross_solution_unittest_reward/std": 0.2220945730805397, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 63 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 831.0, |
| "completions/max_terminated_length": 831.0, |
| "completions/mean_length": 480.3515625, |
| "completions/mean_terminated_length": 480.3515625, |
| "completions/min_length": 271.0, |
| "completions/min_terminated_length": 271.0, |
| "entropy": 0.10076904296875, |
| "epoch": 0.5839749073282008, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03183058649301529, |
| "kl": 0.0033092498779296875, |
| "learning_rate": 7.735032325742355e-06, |
| "loss": 0.0006, |
| "num_tokens": 26177366.0, |
| "reward": 1.4325106143951416, |
| "reward_std": 0.3147215247154236, |
| "rewards/cross_solution_unittest_reward/mean": 0.535786435008049, |
| "rewards/cross_solution_unittest_reward/std": 0.19719675183296204, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 64 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 921.0, |
| "completions/max_terminated_length": 921.0, |
| "completions/mean_length": 476.0442810058594, |
| "completions/mean_terminated_length": 476.0442810058594, |
| "completions/min_length": 207.0, |
| "completions/min_terminated_length": 207.0, |
| "entropy": 0.1140289306640625, |
| "epoch": 0.5930995152552039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.036758169531822205, |
| "kl": 0.005259037017822266, |
| "learning_rate": 7.4578166580651335e-06, |
| "loss": -0.0075, |
| "num_tokens": 26565743.0, |
| "reward": 1.1525869369506836, |
| "reward_std": 0.29967427253723145, |
| "rewards/cross_solution_unittest_reward/mean": 0.44647954404354095, |
| "rewards/cross_solution_unittest_reward/std": 0.30218902230262756, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 65 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.9375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 909.0, |
| "completions/mean_length": 543.9818115234375, |
| "completions/mean_terminated_length": 511.9815979003906, |
| "completions/min_length": 229.0, |
| "completions/min_terminated_length": 229.0, |
| "entropy": 0.1009979248046875, |
| "epoch": 0.602224123182207, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03079250641167164, |
| "kl": 0.00479888916015625, |
| "learning_rate": 7.182674431585703e-06, |
| "loss": 0.0201, |
| "num_tokens": 27046624.0, |
| "reward": 1.330259084701538, |
| "reward_std": 0.28341954946517944, |
| "rewards/cross_solution_unittest_reward/mean": 0.4929102957248688, |
| "rewards/cross_solution_unittest_reward/std": 0.2743557021021843, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 66 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1038.0, |
| "completions/max_terminated_length": 1038.0, |
| "completions/mean_length": 488.4270935058594, |
| "completions/mean_terminated_length": 488.4270935058594, |
| "completions/min_length": 243.0, |
| "completions/min_terminated_length": 243.0, |
| "entropy": 0.096893310546875, |
| "epoch": 0.6113487311092102, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.034310225397348404, |
| "kl": 0.0051801204681396484, |
| "learning_rate": 6.909830056250527e-06, |
| "loss": 0.0019, |
| "num_tokens": 27499308.0, |
| "reward": 1.3934414386749268, |
| "reward_std": 0.35809773206710815, |
| "rewards/cross_solution_unittest_reward/mean": 0.5251588076353073, |
| "rewards/cross_solution_unittest_reward/std": 0.24779481440782547, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 67 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 889.0, |
| "completions/max_terminated_length": 889.0, |
| "completions/mean_length": 519.8125, |
| "completions/mean_terminated_length": 519.8125, |
| "completions/min_length": 272.0, |
| "completions/min_terminated_length": 272.0, |
| "entropy": 0.0970611572265625, |
| "epoch": 0.6204733390362133, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03318794071674347, |
| "kl": 0.003513336181640625, |
| "learning_rate": 6.639506067845698e-06, |
| "loss": -0.0132, |
| "num_tokens": 27901196.0, |
| "reward": 1.1413233280181885, |
| "reward_std": 0.27733054757118225, |
| "rewards/cross_solution_unittest_reward/mean": 0.4484158530831337, |
| "rewards/cross_solution_unittest_reward/std": 0.20082928240299225, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 68 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 879.0, |
| "completions/max_terminated_length": 879.0, |
| "completions/mean_length": 482.9661560058594, |
| "completions/mean_terminated_length": 482.9661560058594, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.1024017333984375, |
| "epoch": 0.6295979469632165, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.041993044316768646, |
| "kl": 0.006025791168212891, |
| "learning_rate": 6.3719229464935915e-06, |
| "loss": -0.0028, |
| "num_tokens": 28332687.0, |
| "reward": 1.1847600936889648, |
| "reward_std": 0.2705700099468231, |
| "rewards/cross_solution_unittest_reward/mean": 0.46874039620161057, |
| "rewards/cross_solution_unittest_reward/std": 0.20285168290138245, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 69 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1062.0, |
| "completions/mean_length": 532.9896240234375, |
| "completions/mean_terminated_length": 525.0234985351562, |
| "completions/min_length": 244.0, |
| "completions/min_terminated_length": 244.0, |
| "entropy": 0.0945587158203125, |
| "epoch": 0.6387225548902196, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0388309545814991, |
| "kl": 0.004973173141479492, |
| "learning_rate": 6.107298936826086e-06, |
| "loss": -0.0019, |
| "num_tokens": 28783291.0, |
| "reward": 1.2814630270004272, |
| "reward_std": 0.39242124557495117, |
| "rewards/cross_solution_unittest_reward/mean": 0.4748888313770294, |
| "rewards/cross_solution_unittest_reward/std": 0.21190231293439865, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 70 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 955.0, |
| "completions/max_terminated_length": 955.0, |
| "completions/mean_length": 495.1302185058594, |
| "completions/mean_terminated_length": 495.1302185058594, |
| "completions/min_length": 257.0, |
| "completions/min_terminated_length": 257.0, |
| "entropy": 0.1051025390625, |
| "epoch": 0.6478471628172227, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03422863036394119, |
| "kl": 0.003699779510498047, |
| "learning_rate": 5.845849869981137e-06, |
| "loss": -0.0108, |
| "num_tokens": 29185853.0, |
| "reward": 0.7699536085128784, |
| "reward_std": 0.227044939994812, |
| "rewards/cross_solution_unittest_reward/mean": 0.33149589970707893, |
| "rewards/cross_solution_unittest_reward/std": 0.19518966227769852, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 71 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1106.0, |
| "completions/mean_length": 521.8333740234375, |
| "completions/mean_terminated_length": 513.838134765625, |
| "completions/min_length": 275.0, |
| "completions/min_terminated_length": 275.0, |
| "entropy": 0.1019439697265625, |
| "epoch": 0.6569717707442259, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.1640857458114624, |
| "kl": 0.015253067016601562, |
| "learning_rate": 5.587788987567785e-06, |
| "loss": -0.0029, |
| "num_tokens": 29597149.0, |
| "reward": 1.2916417121887207, |
| "reward_std": 0.2836645245552063, |
| "rewards/cross_solution_unittest_reward/mean": 0.49249671399593353, |
| "rewards/cross_solution_unittest_reward/std": 0.2682873010635376, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 72 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 772.0, |
| "completions/mean_length": 483.203125, |
| "completions/mean_terminated_length": 475.1070556640625, |
| "completions/min_length": 239.0, |
| "completions/min_terminated_length": 239.0, |
| "entropy": 0.1068267822265625, |
| "epoch": 0.666096378671229, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03074985183775425, |
| "kl": 0.003566741943359375, |
| "learning_rate": 5.333326767743263e-06, |
| "loss": 0.0117, |
| "num_tokens": 30000275.0, |
| "reward": 1.3340082168579102, |
| "reward_std": 0.30908769369125366, |
| "rewards/cross_solution_unittest_reward/mean": 0.5031353682279587, |
| "rewards/cross_solution_unittest_reward/std": 0.23924725502729416, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 73 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1477.0, |
| "completions/max_terminated_length": 1477.0, |
| "completions/mean_length": 533.8151245117188, |
| "completions/mean_terminated_length": 533.8151245117188, |
| "completions/min_length": 211.0, |
| "completions/min_terminated_length": 211.0, |
| "entropy": 0.0978546142578125, |
| "epoch": 0.6752209865982322, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.029303815215826035, |
| "kl": 0.003360271453857422, |
| "learning_rate": 5.082670753543961e-06, |
| "loss": -0.0034, |
| "num_tokens": 30411700.0, |
| "reward": 1.1401095390319824, |
| "reward_std": 0.32691219449043274, |
| "rewards/cross_solution_unittest_reward/mean": 0.4419480115175247, |
| "rewards/cross_solution_unittest_reward/std": 0.24548739194869995, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 74 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1453.0, |
| "completions/max_terminated_length": 1453.0, |
| "completions/mean_length": 520.4713745117188, |
| "completions/mean_terminated_length": 520.4713745117188, |
| "completions/min_length": 268.0, |
| "completions/min_terminated_length": 268.0, |
| "entropy": 0.109771728515625, |
| "epoch": 0.6843455945252352, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03002401813864708, |
| "kl": 0.003072500228881836, |
| "learning_rate": 4.836025383610382e-06, |
| "loss": 0.0128, |
| "num_tokens": 30793049.0, |
| "reward": 1.1908866167068481, |
| "reward_std": 0.269525408744812, |
| "rewards/cross_solution_unittest_reward/mean": 0.46111442148685455, |
| "rewards/cross_solution_unittest_reward/std": 0.23126034438610077, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 75 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1016.0, |
| "completions/max_terminated_length": 1016.0, |
| "completions/mean_length": 458.9505310058594, |
| "completions/mean_terminated_length": 458.9505310058594, |
| "completions/min_length": 241.0, |
| "completions/min_terminated_length": 241.0, |
| "entropy": 0.1105194091796875, |
| "epoch": 0.6934702024522383, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033196672797203064, |
| "kl": 0.003894329071044922, |
| "learning_rate": 4.593591825444028e-06, |
| "loss": 0.0082, |
| "num_tokens": 31135174.0, |
| "reward": 0.9832911491394043, |
| "reward_std": 0.23262672126293182, |
| "rewards/cross_solution_unittest_reward/mean": 0.3907434269785881, |
| "rewards/cross_solution_unittest_reward/std": 0.22558597475290298, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 76 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 998.0, |
| "completions/mean_length": 499.5442810058594, |
| "completions/mean_terminated_length": 491.4908752441406, |
| "completions/min_length": 239.0, |
| "completions/min_terminated_length": 239.0, |
| "entropy": 0.1078948974609375, |
| "epoch": 0.7025948103792415, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033747926354408264, |
| "kl": 0.003387928009033203, |
| "learning_rate": 4.355567811332311e-06, |
| "loss": 0.0017, |
| "num_tokens": 31541479.0, |
| "reward": 1.2079272270202637, |
| "reward_std": 0.29688721895217896, |
| "rewards/cross_solution_unittest_reward/mean": 0.4636748433113098, |
| "rewards/cross_solution_unittest_reward/std": 0.2018030509352684, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 77 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 3241.0, |
| "completions/max_terminated_length": 3241.0, |
| "completions/mean_length": 514.5677490234375, |
| "completions/mean_terminated_length": 514.5677490234375, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.0965728759765625, |
| "epoch": 0.7117194183062446, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03385668992996216, |
| "kl": 0.0037603378295898438, |
| "learning_rate": 4.12214747707527e-06, |
| "loss": 0.0088, |
| "num_tokens": 31984281.0, |
| "reward": 1.2030308246612549, |
| "reward_std": 0.3671332597732544, |
| "rewards/cross_solution_unittest_reward/mean": 0.46230846643447876, |
| "rewards/cross_solution_unittest_reward/std": 0.2557263821363449, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 78 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 3389.0, |
| "completions/mean_length": 526.1927490234375, |
| "completions/mean_terminated_length": 502.115478515625, |
| "completions/min_length": 257.0, |
| "completions/min_terminated_length": 257.0, |
| "entropy": 0.1031036376953125, |
| "epoch": 0.7208440262332478, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0314985066652298, |
| "kl": 0.003326892852783203, |
| "learning_rate": 3.893521203645618e-06, |
| "loss": 0.0163, |
| "num_tokens": 32385115.0, |
| "reward": 1.1673388481140137, |
| "reward_std": 0.2541990876197815, |
| "rewards/cross_solution_unittest_reward/mean": 0.4651763513684273, |
| "rewards/cross_solution_unittest_reward/std": 0.22935150563716888, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 79 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 837.0, |
| "completions/mean_length": 480.5520935058594, |
| "completions/mean_terminated_length": 472.4490966796875, |
| "completions/min_length": 236.0, |
| "completions/min_terminated_length": 236.0, |
| "entropy": 0.100311279296875, |
| "epoch": 0.7299686341602509, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03507440537214279, |
| "kl": 0.004328250885009766, |
| "learning_rate": 3.6698754619112974e-06, |
| "loss": -0.0013, |
| "num_tokens": 32748231.0, |
| "reward": 1.034367561340332, |
| "reward_std": 0.23578712344169617, |
| "rewards/cross_solution_unittest_reward/mean": 0.4301436319947243, |
| "rewards/cross_solution_unittest_reward/std": 0.23155031353235245, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 80 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 898.0, |
| "completions/max_terminated_length": 898.0, |
| "completions/mean_length": 479.953125, |
| "completions/mean_terminated_length": 479.953125, |
| "completions/min_length": 204.0, |
| "completions/min_terminated_length": 204.0, |
| "entropy": 0.1009063720703125, |
| "epoch": 0.739093242087254, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030309055000543594, |
| "kl": 0.0035872459411621094, |
| "learning_rate": 3.4513926605471504e-06, |
| "loss": -0.0042, |
| "num_tokens": 33129445.0, |
| "reward": 1.2114933729171753, |
| "reward_std": 0.22488722205162048, |
| "rewards/cross_solution_unittest_reward/mean": 0.4736487567424774, |
| "rewards/cross_solution_unittest_reward/std": 0.20029103010892868, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 81 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1235.0, |
| "completions/max_terminated_length": 1235.0, |
| "completions/mean_length": 479.26824951171875, |
| "completions/mean_terminated_length": 479.26824951171875, |
| "completions/min_length": 231.0, |
| "completions/min_terminated_length": 231.0, |
| "entropy": 0.105133056640625, |
| "epoch": 0.7482178500142572, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.05099751427769661, |
| "kl": 0.007653713226318359, |
| "learning_rate": 3.2382509972598087e-06, |
| "loss": -0.0018, |
| "num_tokens": 33519820.0, |
| "reward": 1.122490406036377, |
| "reward_std": 0.26781564950942993, |
| "rewards/cross_solution_unittest_reward/mean": 0.4443480148911476, |
| "rewards/cross_solution_unittest_reward/std": 0.24169741570949554, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 82 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 901.0, |
| "completions/mean_length": 487.75262451171875, |
| "completions/mean_terminated_length": 479.6684265136719, |
| "completions/min_length": 281.0, |
| "completions/min_terminated_length": 281.0, |
| "entropy": 0.104339599609375, |
| "epoch": 0.7573424579412603, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.035204458981752396, |
| "kl": 0.004299163818359375, |
| "learning_rate": 3.0306243134470668e-06, |
| "loss": 0.0056, |
| "num_tokens": 33916373.0, |
| "reward": 1.06869375705719, |
| "reward_std": 0.3348367214202881, |
| "rewards/cross_solution_unittest_reward/mean": 0.4094615653157234, |
| "rewards/cross_solution_unittest_reward/std": 0.23972471803426743, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 83 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.890625, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 767.0, |
| "completions/mean_length": 528.9635620117188, |
| "completions/mean_terminated_length": 472.23870849609375, |
| "completions/min_length": 256.0, |
| "completions/min_terminated_length": 256.0, |
| "entropy": 0.10943603515625, |
| "epoch": 0.7664670658682635, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03322676569223404, |
| "kl": 0.0032072067260742188, |
| "learning_rate": 2.8286819524103657e-06, |
| "loss": 0.0142, |
| "num_tokens": 34275463.0, |
| "reward": 1.1465957164764404, |
| "reward_std": 0.2977037727832794, |
| "rewards/cross_solution_unittest_reward/mean": 0.44328781962394714, |
| "rewards/cross_solution_unittest_reward/std": 0.22255534678697586, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 84 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 2205.0, |
| "completions/mean_length": 511.40625, |
| "completions/mean_terminated_length": 503.3838195800781, |
| "completions/min_length": 286.0, |
| "completions/min_terminated_length": 286.0, |
| "entropy": 0.098388671875, |
| "epoch": 0.7755916737952666, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.04618420824408531, |
| "kl": 0.00617527961730957, |
| "learning_rate": 2.6325886212359496e-06, |
| "loss": 0.0117, |
| "num_tokens": 34711467.0, |
| "reward": 1.1300448179244995, |
| "reward_std": 0.23039524257183075, |
| "rewards/cross_solution_unittest_reward/mean": 0.4431014582514763, |
| "rewards/cross_solution_unittest_reward/std": 0.19984686374664307, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 85 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1019.0, |
| "completions/max_terminated_length": 1019.0, |
| "completions/mean_length": 476.16668701171875, |
| "completions/mean_terminated_length": 476.16668701171875, |
| "completions/min_length": 182.0, |
| "completions/min_terminated_length": 182.0, |
| "entropy": 0.1077423095703125, |
| "epoch": 0.7847162817222697, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0404982790350914, |
| "kl": 0.004585742950439453, |
| "learning_rate": 2.4425042564574186e-06, |
| "loss": 0.0068, |
| "num_tokens": 35123059.0, |
| "reward": 1.0940883159637451, |
| "reward_std": 0.3068244159221649, |
| "rewards/cross_solution_unittest_reward/mean": 0.432598315179348, |
| "rewards/cross_solution_unittest_reward/std": 0.20239350944757462, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 86 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.953125, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 2127.0, |
| "completions/mean_length": 533.90625, |
| "completions/mean_terminated_length": 509.8897705078125, |
| "completions/min_length": 228.0, |
| "completions/min_terminated_length": 228.0, |
| "entropy": 0.1087188720703125, |
| "epoch": 0.7938408896492729, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.0433526337146759, |
| "kl": 0.004700660705566406, |
| "learning_rate": 2.2585838936091753e-06, |
| "loss": 0.0154, |
| "num_tokens": 35560487.0, |
| "reward": 1.2438384294509888, |
| "reward_std": 0.2887391448020935, |
| "rewards/cross_solution_unittest_reward/mean": 0.4800245463848114, |
| "rewards/cross_solution_unittest_reward/std": 0.22094998508691788, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 87 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 803.0, |
| "completions/mean_length": 444.28387451171875, |
| "completions/mean_terminated_length": 436.086181640625, |
| "completions/min_length": 216.0, |
| "completions/min_terminated_length": 216.0, |
| "entropy": 0.111419677734375, |
| "epoch": 0.802965497576276, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03479230776429176, |
| "kl": 0.003818035125732422, |
| "learning_rate": 2.0809775407772505e-06, |
| "loss": -0.003, |
| "num_tokens": 35918492.0, |
| "reward": 1.212603211402893, |
| "reward_std": 0.3318028450012207, |
| "rewards/cross_solution_unittest_reward/mean": 0.44981423020362854, |
| "rewards/cross_solution_unittest_reward/std": 0.22631778568029404, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 88 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 880.0, |
| "completions/max_terminated_length": 880.0, |
| "completions/mean_length": 510.78125, |
| "completions/mean_terminated_length": 510.78125, |
| "completions/min_length": 327.0, |
| "completions/min_terminated_length": 327.0, |
| "entropy": 0.0924224853515625, |
| "epoch": 0.8120901055032792, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030438624322414398, |
| "kl": 0.0034804344177246094, |
| "learning_rate": 1.9098300562505266e-06, |
| "loss": 0.0079, |
| "num_tokens": 36344032.0, |
| "reward": 1.1114070415496826, |
| "reward_std": 0.2217615246772766, |
| "rewards/cross_solution_unittest_reward/mean": 0.4325195252895355, |
| "rewards/cross_solution_unittest_reward/std": 0.2371705397963524, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 89 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 875.0, |
| "completions/max_terminated_length": 875.0, |
| "completions/mean_length": 486.8203125, |
| "completions/mean_terminated_length": 486.8203125, |
| "completions/min_length": 248.0, |
| "completions/min_terminated_length": 248.0, |
| "entropy": 0.0999908447265625, |
| "epoch": 0.8212147134302823, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.040998078882694244, |
| "kl": 0.008658885955810547, |
| "learning_rate": 1.74528103037226e-06, |
| "loss": 0.006, |
| "num_tokens": 36757971.0, |
| "reward": 1.045344591140747, |
| "reward_std": 0.34945201873779297, |
| "rewards/cross_solution_unittest_reward/mean": 0.3958388417959213, |
| "rewards/cross_solution_unittest_reward/std": 0.27266910672187805, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 90 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1187.0, |
| "completions/max_terminated_length": 1187.0, |
| "completions/mean_length": 489.68231201171875, |
| "completions/mean_terminated_length": 489.68231201171875, |
| "completions/min_length": 234.0, |
| "completions/min_terminated_length": 234.0, |
| "entropy": 0.105194091796875, |
| "epoch": 0.8303393213572854, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03380502387881279, |
| "kl": 0.003403186798095703, |
| "learning_rate": 1.587464671688187e-06, |
| "loss": 0.0057, |
| "num_tokens": 37152673.0, |
| "reward": 1.0752450227737427, |
| "reward_std": 0.20711544156074524, |
| "rewards/cross_solution_unittest_reward/mean": 0.4331958070397377, |
| "rewards/cross_solution_unittest_reward/std": 0.20185105502605438, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 91 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1024.0, |
| "completions/max_terminated_length": 1024.0, |
| "completions/mean_length": 494.28387451171875, |
| "completions/mean_terminated_length": 494.28387451171875, |
| "completions/min_length": 246.0, |
| "completions/min_terminated_length": 246.0, |
| "entropy": 0.1022796630859375, |
| "epoch": 0.8394639292842886, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.030283644795417786, |
| "kl": 0.0035581588745117188, |
| "learning_rate": 1.436509697484111e-06, |
| "loss": -0.0023, |
| "num_tokens": 37567030.0, |
| "reward": 1.0902509689331055, |
| "reward_std": 0.27206262946128845, |
| "rewards/cross_solution_unittest_reward/mean": 0.42259418964385986, |
| "rewards/cross_solution_unittest_reward/std": 0.244268037378788, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 92 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.96875, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1057.0, |
| "completions/mean_length": 564.1666870117188, |
| "completions/mean_terminated_length": 548.3560180664062, |
| "completions/min_length": 263.0, |
| "completions/min_terminated_length": 263.0, |
| "entropy": 0.0888824462890625, |
| "epoch": 0.8485885372112917, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.1316825151443481, |
| "kl": 0.043515682220458984, |
| "learning_rate": 1.2925392288022299e-06, |
| "loss": 0.0208, |
| "num_tokens": 38002926.0, |
| "reward": 1.0368365049362183, |
| "reward_std": 0.2775520086288452, |
| "rewards/cross_solution_unittest_reward/mean": 0.4123196601867676, |
| "rewards/cross_solution_unittest_reward/std": 0.22595145553350449, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 93 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 973.0, |
| "completions/mean_length": 524.9296875, |
| "completions/mean_terminated_length": 516.9425659179688, |
| "completions/min_length": 227.0, |
| "completions/min_terminated_length": 227.0, |
| "entropy": 0.0940704345703125, |
| "epoch": 0.8577131451382949, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.028070101514458656, |
| "kl": 0.0034241676330566406, |
| "learning_rate": 1.1556706900218572e-06, |
| "loss": -0.0006, |
| "num_tokens": 38420499.0, |
| "reward": 1.074540615081787, |
| "reward_std": 0.32584017515182495, |
| "rewards/cross_solution_unittest_reward/mean": 0.41290927678346634, |
| "rewards/cross_solution_unittest_reward/std": 0.2791058123111725, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 94 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1074.0, |
| "completions/max_terminated_length": 1074.0, |
| "completions/mean_length": 493.265625, |
| "completions/mean_terminated_length": 493.265625, |
| "completions/min_length": 281.0, |
| "completions/min_terminated_length": 281.0, |
| "entropy": 0.09954833984375, |
| "epoch": 0.866837753065298, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03178764134645462, |
| "kl": 0.003660440444946289, |
| "learning_rate": 1.0260157130864178e-06, |
| "loss": 0.0012, |
| "num_tokens": 38866593.0, |
| "reward": 1.128114104270935, |
| "reward_std": 0.27225562930107117, |
| "rewards/cross_solution_unittest_reward/mean": 0.43701624870300293, |
| "rewards/cross_solution_unittest_reward/std": 0.18694043159484863, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 95 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1080.0, |
| "completions/max_terminated_length": 1080.0, |
| "completions/mean_length": 523.0546875, |
| "completions/mean_terminated_length": 523.0546875, |
| "completions/min_length": 295.0, |
| "completions/min_terminated_length": 295.0, |
| "entropy": 0.100311279296875, |
| "epoch": 0.8759623609923011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.032469868659973145, |
| "kl": 0.003574371337890625, |
| "learning_rate": 9.036800464548157e-07, |
| "loss": 0.0029, |
| "num_tokens": 39255286.0, |
| "reward": 1.0960664749145508, |
| "reward_std": 0.25419503450393677, |
| "rewards/cross_solution_unittest_reward/mean": 0.41909460723400116, |
| "rewards/cross_solution_unittest_reward/std": 0.2185814157128334, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 96 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 793.0, |
| "completions/mean_length": 463.96356201171875, |
| "completions/mean_terminated_length": 455.8172302246094, |
| "completions/min_length": 178.0, |
| "completions/min_terminated_length": 178.0, |
| "entropy": 0.1122283935546875, |
| "epoch": 0.8850869689193043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03675765171647072, |
| "kl": 0.0036330223083496094, |
| "learning_rate": 7.887634688515e-07, |
| "loss": 0.0198, |
| "num_tokens": 39578848.0, |
| "reward": 1.2880136966705322, |
| "reward_std": 0.2283666580915451, |
| "rewards/cross_solution_unittest_reward/mean": 0.501230925321579, |
| "rewards/cross_solution_unittest_reward/std": 0.22084086388349533, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 97 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 898.0, |
| "completions/max_terminated_length": 898.0, |
| "completions/mean_length": 530.3646240234375, |
| "completions/mean_terminated_length": 530.3646240234375, |
| "completions/min_length": 325.0, |
| "completions/min_terminated_length": 325.0, |
| "entropy": 0.099212646484375, |
| "epoch": 0.8942115768463074, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03102123737335205, |
| "kl": 0.0036611557006835938, |
| "learning_rate": 6.813597078854772e-07, |
| "loss": -0.0129, |
| "num_tokens": 40025780.0, |
| "reward": 1.2561891078948975, |
| "reward_std": 0.3024594485759735, |
| "rewards/cross_solution_unittest_reward/mean": 0.48492124676704407, |
| "rewards/cross_solution_unittest_reward/std": 0.2765001207590103, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 98 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 998.0, |
| "completions/mean_length": 492.99481201171875, |
| "completions/mean_terminated_length": 484.9242858886719, |
| "completions/min_length": 234.0, |
| "completions/min_terminated_length": 234.0, |
| "entropy": 0.103118896484375, |
| "epoch": 0.9033361847733106, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031520117074251175, |
| "kl": 0.003575563430786133, |
| "learning_rate": 5.815563636047539e-07, |
| "loss": -0.0028, |
| "num_tokens": 40397250.0, |
| "reward": 1.2597386837005615, |
| "reward_std": 0.2905423939228058, |
| "rewards/cross_solution_unittest_reward/mean": 0.4819524586200714, |
| "rewards/cross_solution_unittest_reward/std": 0.21879704296588898, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 99 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 928.0, |
| "completions/max_terminated_length": 928.0, |
| "completions/mean_length": 472.08856201171875, |
| "completions/mean_terminated_length": 472.08856201171875, |
| "completions/min_length": 201.0, |
| "completions/min_terminated_length": 201.0, |
| "entropy": 0.11407470703125, |
| "epoch": 0.9124607927003137, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.033804018050432205, |
| "kl": 0.003757953643798828, |
| "learning_rate": 4.894348370484648e-07, |
| "loss": -0.0169, |
| "num_tokens": 40820412.0, |
| "reward": 1.1143543720245361, |
| "reward_std": 0.2880322337150574, |
| "rewards/cross_solution_unittest_reward/mean": 0.42812392115592957, |
| "rewards/cross_solution_unittest_reward/std": 0.24535022675991058, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 100 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 891.0, |
| "completions/max_terminated_length": 891.0, |
| "completions/mean_length": 476.58074951171875, |
| "completions/mean_terminated_length": 476.58074951171875, |
| "completions/min_length": 262.0, |
| "completions/min_terminated_length": 262.0, |
| "entropy": 0.09490966796875, |
| "epoch": 0.9215854006273168, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.031989987939596176, |
| "kl": 0.0035381317138671875, |
| "learning_rate": 4.0507026385502747e-07, |
| "loss": 0.0016, |
| "num_tokens": 41205987.0, |
| "reward": 1.1675355434417725, |
| "reward_std": 0.27103692293167114, |
| "rewards/cross_solution_unittest_reward/mean": 0.4633182883262634, |
| "rewards/cross_solution_unittest_reward/std": 0.19777817279100418, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 101 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 886.0, |
| "completions/mean_length": 509.75262451171875, |
| "completions/mean_terminated_length": 501.7258605957031, |
| "completions/min_length": 240.0, |
| "completions/min_terminated_length": 240.0, |
| "entropy": 0.09368896484375, |
| "epoch": 0.93071000855432, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.06643658876419067, |
| "kl": 0.004868507385253906, |
| "learning_rate": 3.2853145298042954e-07, |
| "loss": 0.0168, |
| "num_tokens": 41637052.0, |
| "reward": 1.360771894454956, |
| "reward_std": 0.33854445815086365, |
| "rewards/cross_solution_unittest_reward/mean": 0.5105735212564468, |
| "rewards/cross_solution_unittest_reward/std": 0.19905748218297958, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 102 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 857.0, |
| "completions/max_terminated_length": 857.0, |
| "completions/mean_length": 468.79949951171875, |
| "completions/mean_terminated_length": 468.79949951171875, |
| "completions/min_length": 238.0, |
| "completions/min_terminated_length": 238.0, |
| "entropy": 0.1016998291015625, |
| "epoch": 0.939834616481323, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.044055838137865067, |
| "kl": 0.003973722457885742, |
| "learning_rate": 2.5988083057666534e-07, |
| "loss": 0.0033, |
| "num_tokens": 42007991.0, |
| "reward": 1.3666280508041382, |
| "reward_std": 0.3037351369857788, |
| "rewards/cross_solution_unittest_reward/mean": 0.5158256590366364, |
| "rewards/cross_solution_unittest_reward/std": 0.24741099774837494, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 103 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 921.0, |
| "completions/max_terminated_length": 921.0, |
| "completions/mean_length": 493.125, |
| "completions/mean_terminated_length": 493.125, |
| "completions/min_length": 297.0, |
| "completions/min_terminated_length": 297.0, |
| "entropy": 0.09735107421875, |
| "epoch": 0.9489592244083263, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03200039267539978, |
| "kl": 0.004022121429443359, |
| "learning_rate": 1.9917438907606556e-07, |
| "loss": -0.0008, |
| "num_tokens": 42423575.0, |
| "reward": 1.1082267761230469, |
| "reward_std": 0.2577505111694336, |
| "rewards/cross_solution_unittest_reward/mean": 0.4455152079463005, |
| "rewards/cross_solution_unittest_reward/std": 0.19009340554475784, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 104 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 3546.0, |
| "completions/max_terminated_length": 3546.0, |
| "completions/mean_length": 478.2317810058594, |
| "completions/mean_terminated_length": 478.2317810058594, |
| "completions/min_length": 273.0, |
| "completions/min_terminated_length": 273.0, |
| "entropy": 0.10748291015625, |
| "epoch": 0.9580838323353293, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03472354635596275, |
| "kl": 0.003666400909423828, |
| "learning_rate": 1.464616415230702e-07, |
| "loss": 0.0101, |
| "num_tokens": 42791328.0, |
| "reward": 0.8813101649284363, |
| "reward_std": 0.30557072162628174, |
| "rewards/cross_solution_unittest_reward/mean": 0.3544023931026459, |
| "rewards/cross_solution_unittest_reward/std": 0.22855371981859207, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 105 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -4.984375, |
| "completions/max_length": 3584.0, |
| "completions/max_terminated_length": 1037.0, |
| "completions/mean_length": 535.1146240234375, |
| "completions/mean_terminated_length": 527.154052734375, |
| "completions/min_length": 314.0, |
| "completions/min_terminated_length": 314.0, |
| "entropy": 0.09112548828125, |
| "epoch": 0.9672084402623324, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.027567479759454727, |
| "kl": 0.0032286643981933594, |
| "learning_rate": 1.0178558119067316e-07, |
| "loss": -0.0102, |
| "num_tokens": 43222180.0, |
| "reward": 1.1219594478607178, |
| "reward_std": 0.34339237213134766, |
| "rewards/cross_solution_unittest_reward/mean": 0.4241596907377243, |
| "rewards/cross_solution_unittest_reward/std": 0.2556988596916199, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 106 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 840.0, |
| "completions/max_terminated_length": 840.0, |
| "completions/mean_length": 493.3828125, |
| "completions/mean_terminated_length": 493.3828125, |
| "completions/min_length": 253.0, |
| "completions/min_terminated_length": 253.0, |
| "entropy": 0.112396240234375, |
| "epoch": 0.9763330481893356, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.036228280514478683, |
| "kl": 0.004697322845458984, |
| "learning_rate": 6.51826465144978e-08, |
| "loss": -0.0014, |
| "num_tokens": 43626183.0, |
| "reward": 1.1601941585540771, |
| "reward_std": 0.27455127239227295, |
| "rewards/cross_solution_unittest_reward/mean": 0.457772821187973, |
| "rewards/cross_solution_unittest_reward/std": 0.23431915044784546, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 107 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 1087.0, |
| "completions/max_terminated_length": 1087.0, |
| "completions/mean_length": 527.7291870117188, |
| "completions/mean_terminated_length": 527.7291870117188, |
| "completions/min_length": 233.0, |
| "completions/min_terminated_length": 233.0, |
| "entropy": 0.0887908935546875, |
| "epoch": 0.9854576561163387, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.02786332555115223, |
| "kl": 0.002989053726196289, |
| "learning_rate": 3.668269137308666e-08, |
| "loss": -0.005, |
| "num_tokens": 44066847.0, |
| "reward": 1.3689097166061401, |
| "reward_std": 0.2395750731229782, |
| "rewards/cross_solution_unittest_reward/mean": 0.5246002376079559, |
| "rewards/cross_solution_unittest_reward/std": 0.2055259644985199, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 108 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": -5.0, |
| "completions/max_length": 819.0, |
| "completions/max_terminated_length": 819.0, |
| "completions/mean_length": 473.96356201171875, |
| "completions/mean_terminated_length": 473.96356201171875, |
| "completions/min_length": 202.0, |
| "completions/min_terminated_length": 202.0, |
| "entropy": 0.1051177978515625, |
| "epoch": 0.9945822640433419, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.03451412171125412, |
| "kl": 0.0034637451171875, |
| "learning_rate": 1.630896073864352e-08, |
| "loss": 0.0043, |
| "num_tokens": 44441753.0, |
| "reward": 1.357940435409546, |
| "reward_std": 0.3025989532470703, |
| "rewards/cross_solution_unittest_reward/mean": 0.5211868733167648, |
| "rewards/cross_solution_unittest_reward/std": 0.2235267534852028, |
| "rewards/format_reward/mean": 0.0, |
| "rewards/format_reward/std": 0.0, |
| "step": 109 |
| }, |
| { |
| "epoch": 0.9945822640433419, |
| "step": 109, |
| "total_flos": 0.0, |
| "train_loss": 0.002528898397298306, |
| "train_runtime": 42447.4995, |
| "train_samples_per_second": 0.031, |
| "train_steps_per_second": 0.003 |
| } |
| ], |
| "logging_steps": 1, |
| "max_steps": 110, |
| "num_input_tokens_seen": 44441753, |
| "num_train_epochs": 1, |
| "save_steps": 25, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": false |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 2, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
