{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9945822640433419, "eval_steps": 500, "global_step": 109, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 525.9375, "completions/mean_terminated_length": 525.9375, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.110321044921875, "epoch": 0.009124607927003137, "frac_reward_zero_std": 0.0, "grad_norm": 0.025090789422392845, "kl": 0.0002155303955078125, "learning_rate": 2e-05, "loss": 0.002, "num_tokens": 410568.0, "reward": 1.2271196842193604, "reward_std": 0.31988272070884705, "rewards/cross_solution_unittest_reward/mean": 0.47540418803691864, "rewards/cross_solution_unittest_reward/std": 0.23665796220302582, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 1 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1937.0, "completions/max_terminated_length": 1937.0, "completions/mean_length": 493.22918701171875, "completions/mean_terminated_length": 493.22918701171875, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.1114044189453125, "epoch": 0.018249215854006275, "frac_reward_zero_std": 0.0, "grad_norm": 0.028324643149971962, "kl": 0.00023484230041503906, "learning_rate": 1.9995921928281893e-05, "loss": -0.002, "num_tokens": 821504.0, "reward": 1.0042212009429932, "reward_std": 0.2644613981246948, "rewards/cross_solution_unittest_reward/mean": 0.39199119806289673, "rewards/cross_solution_unittest_reward/std": 0.25940997898578644, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 2 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1133.0, "completions/max_terminated_length": 1133.0, "completions/mean_length": 567.2474365234375, "completions/mean_terminated_length": 567.2474365234375, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.0968017578125, "epoch": 0.02737382378100941, "frac_reward_zero_std": 0.0, "grad_norm": 0.02467537671327591, "kl": 0.00020578503608703613, "learning_rate": 1.9983691039261358e-05, "loss": 0.009, "num_tokens": 1260095.0, "reward": 1.0529532432556152, "reward_std": 0.2820073366165161, "rewards/cross_solution_unittest_reward/mean": 0.407609261572361, "rewards/cross_solution_unittest_reward/std": 0.20918594300746918, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 3 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 779.0, "completions/max_terminated_length": 779.0, "completions/mean_length": 484.875, "completions/mean_terminated_length": 484.875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.1236572265625, "epoch": 0.03649843170801255, "frac_reward_zero_std": 0.0, "grad_norm": 0.04513426125049591, "kl": 0.0002601146697998047, "learning_rate": 1.9963317308626916e-05, "loss": -0.0077, "num_tokens": 1693503.0, "reward": 0.7472698092460632, "reward_std": 0.2714373469352722, "rewards/cross_solution_unittest_reward/mean": 0.30792947113513947, "rewards/cross_solution_unittest_reward/std": 0.1790020987391472, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 4 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 2220.0, "completions/max_terminated_length": 2220.0, "completions/mean_length": 490.8828125, "completions/mean_terminated_length": 490.8828125, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.123046875, "epoch": 0.04562303963501568, "frac_reward_zero_std": 0.0, "grad_norm": 0.029334085062146187, "kl": 0.0002612471580505371, "learning_rate": 1.99348173534855e-05, "loss": -0.0056, "num_tokens": 2076626.0, "reward": 1.1481083631515503, "reward_std": 0.3634147047996521, "rewards/cross_solution_unittest_reward/mean": 0.44348418712615967, "rewards/cross_solution_unittest_reward/std": 0.280821830034256, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 5 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 1143.0, "completions/mean_length": 545.15625, "completions/mean_terminated_length": 529.24609375, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.1153717041015625, "epoch": 0.05474764756201882, "frac_reward_zero_std": 0.0, "grad_norm": 0.027162108570337296, "kl": 0.0002492964267730713, "learning_rate": 1.989821441880933e-05, "loss": -0.0044, "num_tokens": 2500214.0, "reward": 1.075005054473877, "reward_std": 0.26096874475479126, "rewards/cross_solution_unittest_reward/mean": 0.41980794072151184, "rewards/cross_solution_unittest_reward/std": 0.24784592539072037, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 6 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1106.0, "completions/mean_length": 534.25, "completions/mean_terminated_length": 526.2872314453125, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "entropy": 0.1142120361328125, "epoch": 0.06387225548902195, "frac_reward_zero_std": 0.0, "grad_norm": 0.026346901431679726, "kl": 0.00027573108673095703, "learning_rate": 1.9853538358476933e-05, "loss": -0.0033, "num_tokens": 2945870.0, "reward": 1.0205891132354736, "reward_std": 0.3387899398803711, "rewards/cross_solution_unittest_reward/mean": 0.4038037732243538, "rewards/cross_solution_unittest_reward/std": 0.2187812626361847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 7 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1321.0, "completions/max_terminated_length": 1321.0, "completions/mean_length": 525.65625, "completions/mean_terminated_length": 525.65625, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.109466552734375, "epoch": 0.0729968634160251, "frac_reward_zero_std": 0.0, "grad_norm": 0.027590544894337654, "kl": 0.00027805566787719727, "learning_rate": 1.9800825610923937e-05, "loss": 0.0054, "num_tokens": 3402378.0, "reward": 1.002925157546997, "reward_std": 0.2862505316734314, "rewards/cross_solution_unittest_reward/mean": 0.3803572356700897, "rewards/cross_solution_unittest_reward/std": 0.26964404433965683, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 8 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 950.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 490.0833435058594, "completions/mean_terminated_length": 490.0833435058594, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.1181488037109375, "epoch": 0.08212147134302823, "frac_reward_zero_std": 0.0, "grad_norm": 0.030266448855400085, "kl": 0.00028058886528015137, "learning_rate": 1.9740119169423337e-05, "loss": 0.0072, "num_tokens": 3794010.0, "reward": 1.008456826210022, "reward_std": 0.33921265602111816, "rewards/cross_solution_unittest_reward/mean": 0.4010137543082237, "rewards/cross_solution_unittest_reward/std": 0.22554735094308853, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 9 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1135.0, "completions/max_terminated_length": 1135.0, "completions/mean_length": 505.63543701171875, "completions/mean_terminated_length": 505.63543701171875, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "entropy": 0.1077117919921875, "epoch": 0.09124607927003137, "frac_reward_zero_std": 0.0, "grad_norm": 0.028971431776881218, "kl": 0.0002878904342651367, "learning_rate": 1.9671468547019575e-05, "loss": 0.004, "num_tokens": 4238414.0, "reward": 1.0463480949401855, "reward_std": 0.30459851026535034, "rewards/cross_solution_unittest_reward/mean": 0.40890535712242126, "rewards/cross_solution_unittest_reward/std": 0.23788339644670486, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 10 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1150.0, "completions/mean_length": 543.4557495117188, "completions/mean_terminated_length": 535.5169677734375, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.102996826171875, "epoch": 0.1003706871970345, "frac_reward_zero_std": 0.0, "grad_norm": 0.027505753561854362, "kl": 0.0003129541873931885, "learning_rate": 1.9594929736144978e-05, "loss": 0.0015, "num_tokens": 4662925.0, "reward": 1.0352768898010254, "reward_std": 0.2669483423233032, "rewards/cross_solution_unittest_reward/mean": 0.4138600826263428, "rewards/cross_solution_unittest_reward/std": 0.20296787470579147, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 11 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 1080.0, "completions/mean_length": 506.15887451171875, "completions/mean_terminated_length": 490.04449462890625, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.1264190673828125, "epoch": 0.10949529512403763, "frac_reward_zero_std": 0.0, "grad_norm": 0.031232472509145737, "kl": 0.0004932880401611328, "learning_rate": 1.9510565162951538e-05, "loss": 0.0096, "num_tokens": 5095338.0, "reward": 0.8861899375915527, "reward_std": 0.31088966131210327, "rewards/cross_solution_unittest_reward/mean": 0.34614112228155136, "rewards/cross_solution_unittest_reward/std": 0.22473014146089554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 12 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1216.0, "completions/max_terminated_length": 1216.0, "completions/mean_length": 519.7396240234375, "completions/mean_terminated_length": 519.7396240234375, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "entropy": 0.112884521484375, "epoch": 0.11861990305104078, "frac_reward_zero_std": 0.0, "grad_norm": 0.0282323956489563, "kl": 0.00036662817001342773, "learning_rate": 1.941844363639525e-05, "loss": -0.0078, "num_tokens": 5502462.0, "reward": 1.0762056112289429, "reward_std": 0.25708380341529846, "rewards/cross_solution_unittest_reward/mean": 0.4215054139494896, "rewards/cross_solution_unittest_reward/std": 0.23641275614500046, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 13 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 2663.0, "completions/max_terminated_length": 2663.0, "completions/mean_length": 620.8854370117188, "completions/mean_terminated_length": 620.8854370117188, "completions/min_length": 270.0, "completions/min_terminated_length": 270.0, "entropy": 0.1026458740234375, "epoch": 0.1277445109780439, "frac_reward_zero_std": 0.0, "grad_norm": 0.026703201234340668, "kl": 0.00039651989936828613, "learning_rate": 1.9318640292114526e-05, "loss": 0.0004, "num_tokens": 5972986.0, "reward": 1.1370387077331543, "reward_std": 0.24530315399169922, "rewards/cross_solution_unittest_reward/mean": 0.4404096007347107, "rewards/cross_solution_unittest_reward/std": 0.2456706538796425, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 14 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, "completions/max_terminated_length": 934.0, "completions/mean_length": 482.0364685058594, "completions/mean_terminated_length": 457.6115417480469, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.126495361328125, "epoch": 0.13686911890504705, "frac_reward_zero_std": 0.0, "grad_norm": 0.029596447944641113, "kl": 0.0004467964172363281, "learning_rate": 1.92112365311485e-05, "loss": 0.0147, "num_tokens": 6324176.0, "reward": 1.1108325719833374, "reward_std": 0.2962481379508972, "rewards/cross_solution_unittest_reward/mean": 0.4290347397327423, "rewards/cross_solution_unittest_reward/std": 0.24357028305530548, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 15 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 2795.0, "completions/mean_length": 522.7630615234375, "completions/mean_terminated_length": 514.770263671875, "completions/min_length": 192.0, "completions/min_terminated_length": 192.0, "entropy": 0.1092681884765625, "epoch": 0.1459937268320502, "frac_reward_zero_std": 0.0, "grad_norm": 0.028343815356492996, "kl": 0.00048470497131347656, "learning_rate": 1.9096319953545186e-05, "loss": -0.0025, "num_tokens": 6724061.0, "reward": 1.055526614189148, "reward_std": 0.34647518396377563, "rewards/cross_solution_unittest_reward/mean": 0.41471436619758606, "rewards/cross_solution_unittest_reward/std": 0.21236862987279892, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 16 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1336.0, "completions/max_terminated_length": 1336.0, "completions/mean_length": 508.3984375, "completions/mean_terminated_length": 508.3984375, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.1138763427734375, "epoch": 0.15511833475905332, "frac_reward_zero_std": 0.0, "grad_norm": 0.028925545513629913, "kl": 0.0004659891128540039, "learning_rate": 1.8973984286913584e-05, "loss": -0.0014, "num_tokens": 7133206.0, "reward": 1.2844151258468628, "reward_std": 0.3014547526836395, "rewards/cross_solution_unittest_reward/mean": 0.473222941160202, "rewards/cross_solution_unittest_reward/std": 0.282610684633255, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 17 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1300.0, "completions/mean_length": 531.9765625, "completions/mean_terminated_length": 524.0078125, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.108856201171875, "epoch": 0.16424294268605646, "frac_reward_zero_std": 0.0, "grad_norm": 0.02900790423154831, "kl": 0.0005418062210083008, "learning_rate": 1.8844329309978146e-05, "loss": 0.0016, "num_tokens": 7567925.0, "reward": 1.0913987159729004, "reward_std": 0.34041401743888855, "rewards/cross_solution_unittest_reward/mean": 0.41306161880493164, "rewards/cross_solution_unittest_reward/std": 0.24276886880397797, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 18 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 906.0, "completions/max_terminated_length": 906.0, "completions/mean_length": 493.9817810058594, "completions/mean_terminated_length": 493.9817810058594, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.1160125732421875, "epoch": 0.17336755061305958, "frac_reward_zero_std": 0.0, "grad_norm": 0.029141006991267204, "kl": 0.000545203685760498, "learning_rate": 1.8707460771197773e-05, "loss": 0.0073, "num_tokens": 7976102.0, "reward": 1.0161412954330444, "reward_std": 0.2866203784942627, "rewards/cross_solution_unittest_reward/mean": 0.39064040780067444, "rewards/cross_solution_unittest_reward/std": 0.2544485256075859, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 19 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1267.0, "completions/max_terminated_length": 1267.0, "completions/mean_length": 527.3229370117188, "completions/mean_terminated_length": 527.3229370117188, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "entropy": 0.1096038818359375, "epoch": 0.18249215854006273, "frac_reward_zero_std": 0.0, "grad_norm": 0.02903016284108162, "kl": 0.0006221532821655273, "learning_rate": 1.856349030251589e-05, "loss": 0.0037, "num_tokens": 8428962.0, "reward": 1.0079458951950073, "reward_std": 0.35072073340415955, "rewards/cross_solution_unittest_reward/mean": 0.38429319858551025, "rewards/cross_solution_unittest_reward/std": 0.22937388718128204, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 20 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1208.0, "completions/max_terminated_length": 1208.0, "completions/mean_length": 482.62762451171875, "completions/mean_terminated_length": 482.62762451171875, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "entropy": 0.109039306640625, "epoch": 0.19161676646706588, "frac_reward_zero_std": 0.0, "grad_norm": 0.02826613187789917, "kl": 0.000597834587097168, "learning_rate": 1.8412535328311813e-05, "loss": 0.0041, "num_tokens": 8812995.0, "reward": 1.0695934295654297, "reward_std": 0.25936269760131836, "rewards/cross_solution_unittest_reward/mean": 0.42481929063796997, "rewards/cross_solution_unittest_reward/std": 0.20526525378227234, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 21 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1185.0, "completions/max_terminated_length": 1185.0, "completions/mean_length": 505.578125, "completions/mean_terminated_length": 505.578125, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "entropy": 0.11138916015625, "epoch": 0.200741374394069, "frac_reward_zero_std": 0.0, "grad_norm": 0.03168891742825508, "kl": 0.0010138154029846191, "learning_rate": 1.825471896962774e-05, "loss": -0.0122, "num_tokens": 9188513.0, "reward": 1.0664725303649902, "reward_std": 0.3464021384716034, "rewards/cross_solution_unittest_reward/mean": 0.40388014912605286, "rewards/cross_solution_unittest_reward/std": 0.2562775984406471, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 22 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1357.0, "completions/max_terminated_length": 1357.0, "completions/mean_length": 507.8411560058594, "completions/mean_terminated_length": 507.8411560058594, "completions/min_length": 186.0, "completions/min_terminated_length": 186.0, "entropy": 0.1090545654296875, "epoch": 0.20986598232107215, "frac_reward_zero_std": 0.0, "grad_norm": 0.030050111934542656, "kl": 0.0007647275924682617, "learning_rate": 1.8090169943749477e-05, "loss": -0.0017, "num_tokens": 9635068.0, "reward": 1.2008159160614014, "reward_std": 0.25730741024017334, "rewards/cross_solution_unittest_reward/mean": 0.4557904154062271, "rewards/cross_solution_unittest_reward/std": 0.22673149406909943, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 23 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 950.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 500.6302185058594, "completions/mean_terminated_length": 500.6302185058594, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.1072540283203125, "epoch": 0.21899059024807527, "frac_reward_zero_std": 0.0, "grad_norm": 0.027480732649564743, "kl": 0.0007421970367431641, "learning_rate": 1.7919022459222754e-05, "loss": 0.0006, "num_tokens": 10054982.0, "reward": 0.9477907419204712, "reward_std": 0.3180457353591919, "rewards/cross_solution_unittest_reward/mean": 0.3783436641097069, "rewards/cross_solution_unittest_reward/std": 0.25479088723659515, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 24 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1137.0, "completions/max_terminated_length": 1137.0, "completions/mean_length": 490.1536560058594, "completions/mean_terminated_length": 490.1536560058594, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.1211090087890625, "epoch": 0.22811519817507842, "frac_reward_zero_std": 0.0, "grad_norm": 0.03646322712302208, "kl": 0.0009087324142456055, "learning_rate": 1.7741416106390828e-05, "loss": 0.0011, "num_tokens": 10456873.0, "reward": 1.0844480991363525, "reward_std": 0.2816399335861206, "rewards/cross_solution_unittest_reward/mean": 0.4169381707906723, "rewards/cross_solution_unittest_reward/std": 0.2162943333387375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 25 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1184.0, "completions/mean_length": 508.9739685058594, "completions/mean_terminated_length": 500.9451904296875, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.1131439208984375, "epoch": 0.23723980610208156, "frac_reward_zero_std": 0.0, "grad_norm": 0.027677049860358238, "kl": 0.0007776618003845215, "learning_rate": 1.7557495743542586e-05, "loss": -0.0092, "num_tokens": 10819487.0, "reward": 0.9719910621643066, "reward_std": 0.28075867891311646, "rewards/cross_solution_unittest_reward/mean": 0.3785587027668953, "rewards/cross_solution_unittest_reward/std": 0.24748405069112778, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 26 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 725.0, "completions/max_terminated_length": 725.0, "completions/mean_length": 460.6380310058594, "completions/mean_terminated_length": 460.6380310058594, "completions/min_length": 209.0, "completions/min_terminated_length": 209.0, "entropy": 0.1148834228515625, "epoch": 0.24636441402908468, "frac_reward_zero_std": 0.0, "grad_norm": 0.028393210843205452, "kl": 0.0009175539016723633, "learning_rate": 1.736741137876405e-05, "loss": -0.0004, "num_tokens": 11193260.0, "reward": 1.155346155166626, "reward_std": 0.2796667814254761, "rewards/cross_solution_unittest_reward/mean": 0.45263613760471344, "rewards/cross_solution_unittest_reward/std": 0.21176424622535706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 27 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 966.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 493.7942810058594, "completions/mean_terminated_length": 493.7942810058594, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.108154296875, "epoch": 0.2554890219560878, "frac_reward_zero_std": 0.0, "grad_norm": 0.02867596223950386, "kl": 0.001038193702697754, "learning_rate": 1.7171318047589637e-05, "loss": 0.0046, "num_tokens": 11621069.0, "reward": 1.267564296722412, "reward_std": 0.28822484612464905, "rewards/cross_solution_unittest_reward/mean": 0.4844939559698105, "rewards/cross_solution_unittest_reward/std": 0.26973531395196915, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 28 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 489.83074951171875, "completions/mean_terminated_length": 481.751953125, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.1177520751953125, "epoch": 0.26461362988309095, "frac_reward_zero_std": 0.0, "grad_norm": 0.030956774950027466, "kl": 0.0012102127075195312, "learning_rate": 1.696937568655294e-05, "loss": -0.0038, "num_tokens": 12017156.0, "reward": 1.182539701461792, "reward_std": 0.231126606464386, "rewards/cross_solution_unittest_reward/mean": 0.4465182274580002, "rewards/cross_solution_unittest_reward/std": 0.28548523783683777, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 29 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 907.0, "completions/mean_length": 530.5963745117188, "completions/mean_terminated_length": 522.6240234375, "completions/min_length": 225.0, "completions/min_terminated_length": 225.0, "entropy": 0.1051483154296875, "epoch": 0.2737382378100941, "frac_reward_zero_std": 0.0, "grad_norm": 0.030249865725636482, "kl": 0.0010318756103515625, "learning_rate": 1.6761749002740195e-05, "loss": -0.0066, "num_tokens": 12439785.0, "reward": 0.9416704177856445, "reward_std": 0.31163209676742554, "rewards/cross_solution_unittest_reward/mean": 0.37286702543497086, "rewards/cross_solution_unittest_reward/std": 0.26768606156110764, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 30 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 832.0, "completions/max_terminated_length": 832.0, "completions/mean_length": 461.34637451171875, "completions/mean_terminated_length": 461.34637451171875, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.11480712890625, "epoch": 0.28286284573709725, "frac_reward_zero_std": 0.0, "grad_norm": 0.03138723969459534, "kl": 0.0012290477752685547, "learning_rate": 1.6548607339452853e-05, "loss": 0.0012, "num_tokens": 12800374.0, "reward": 1.142934799194336, "reward_std": 0.2438894510269165, "rewards/cross_solution_unittest_reward/mean": 0.4574270248413086, "rewards/cross_solution_unittest_reward/std": 0.20738767087459564, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 31 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 954.0, "completions/max_terminated_length": 954.0, "completions/mean_length": 484.15106201171875, "completions/mean_terminated_length": 484.15106201171875, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "entropy": 0.10296630859375, "epoch": 0.2919874536641004, "frac_reward_zero_std": 0.0, "grad_norm": 0.037726812064647675, "kl": 0.0012822151184082031, "learning_rate": 1.6330124538088705e-05, "loss": 0.0074, "num_tokens": 13254672.0, "reward": 1.2426599264144897, "reward_std": 0.39957040548324585, "rewards/cross_solution_unittest_reward/mean": 0.4713882952928543, "rewards/cross_solution_unittest_reward/std": 0.23567520081996918, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 32 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 997.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 474.6171875, "completions/mean_terminated_length": 474.6171875, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.1254119873046875, "epoch": 0.3011120615911035, "frac_reward_zero_std": 0.0, "grad_norm": 0.03023543767631054, "kl": 0.0014002323150634766, "learning_rate": 1.6106478796354382e-05, "loss": -0.0059, "num_tokens": 13634901.0, "reward": 1.0120811462402344, "reward_std": 0.24670013785362244, "rewards/cross_solution_unittest_reward/mean": 0.4261118918657303, "rewards/cross_solution_unittest_reward/std": 0.2368394210934639, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 33 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 880.0, "completions/max_terminated_length": 880.0, "completions/mean_length": 492.13543701171875, "completions/mean_terminated_length": 492.13543701171875, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.1062469482421875, "epoch": 0.31023666951810663, "frac_reward_zero_std": 0.0, "grad_norm": 0.030524957925081253, "kl": 0.0013687610626220703, "learning_rate": 1.5877852522924733e-05, "loss": -0.006, "num_tokens": 14063145.0, "reward": 1.219003438949585, "reward_std": 0.2584400475025177, "rewards/cross_solution_unittest_reward/mean": 0.4796546399593353, "rewards/cross_solution_unittest_reward/std": 0.23066769540309906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 34 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 885.0, "completions/max_terminated_length": 885.0, "completions/mean_length": 498.69012451171875, "completions/mean_terminated_length": 498.69012451171875, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "entropy": 0.1126251220703125, "epoch": 0.3193612774451098, "frac_reward_zero_std": 0.0, "grad_norm": 0.030774278566241264, "kl": 0.0014193058013916016, "learning_rate": 1.5644432188667695e-05, "loss": 0.0139, "num_tokens": 14445962.0, "reward": 1.1249631643295288, "reward_std": 0.32856079936027527, "rewards/cross_solution_unittest_reward/mean": 0.4171721339225769, "rewards/cross_solution_unittest_reward/std": 0.24577928334474564, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 35 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1494.0, "completions/max_terminated_length": 1494.0, "completions/mean_length": 512.0911865234375, "completions/mean_terminated_length": 512.0911865234375, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "entropy": 0.1166534423828125, "epoch": 0.32848588537211293, "frac_reward_zero_std": 0.0, "grad_norm": 0.03613327443599701, "kl": 0.002481698989868164, "learning_rate": 1.5406408174555978e-05, "loss": 0.0038, "num_tokens": 14864613.0, "reward": 1.0395325422286987, "reward_std": 0.31884095072746277, "rewards/cross_solution_unittest_reward/mean": 0.3926886022090912, "rewards/cross_solution_unittest_reward/std": 0.22794856131076813, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 36 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1651.0, "completions/max_terminated_length": 1651.0, "completions/mean_length": 516.1224365234375, "completions/mean_terminated_length": 516.1224365234375, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.0965118408203125, "epoch": 0.3376104932991161, "frac_reward_zero_std": 0.0, "grad_norm": 0.029742714017629623, "kl": 0.0016620159149169922, "learning_rate": 1.5163974616389621e-05, "loss": 0.0001, "num_tokens": 15273692.0, "reward": 1.0600379705429077, "reward_std": 0.26911425590515137, "rewards/cross_solution_unittest_reward/mean": 0.42146605998277664, "rewards/cross_solution_unittest_reward/std": 0.23059702664613724, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 37 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 818.0, "completions/mean_length": 496.34637451171875, "completions/mean_terminated_length": 480.1806335449219, "completions/min_length": 193.0, "completions/min_terminated_length": 193.0, "entropy": 0.110321044921875, "epoch": 0.34673510122611917, "frac_reward_zero_std": 0.0, "grad_norm": 0.030567364767193794, "kl": 0.0016117095947265625, "learning_rate": 1.491732924645604e-05, "loss": 0.0136, "num_tokens": 15662473.0, "reward": 1.1122747659683228, "reward_std": 0.26910364627838135, "rewards/cross_solution_unittest_reward/mean": 0.4311128109693527, "rewards/cross_solution_unittest_reward/std": 0.23016689717769623, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 38 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 962.0, "completions/max_terminated_length": 962.0, "completions/mean_length": 473.0859375, "completions/mean_terminated_length": 473.0859375, "completions/min_length": 162.0, "completions/min_terminated_length": 162.0, "entropy": 0.117919921875, "epoch": 0.3558597091531223, "frac_reward_zero_std": 0.0, "grad_norm": 0.03569599241018295, "kl": 0.002121448516845703, "learning_rate": 1.4666673232256738e-05, "loss": 0.0213, "num_tokens": 15974274.0, "reward": 0.9712950587272644, "reward_std": 0.37336552143096924, "rewards/cross_solution_unittest_reward/mean": 0.37921562045812607, "rewards/cross_solution_unittest_reward/std": 0.21440216898918152, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 39 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1488.0, "completions/max_terminated_length": 1488.0, "completions/mean_length": 496.86981201171875, "completions/mean_terminated_length": 496.86981201171875, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.104644775390625, "epoch": 0.36498431708012546, "frac_reward_zero_std": 0.0, "grad_norm": 0.03822171315550804, "kl": 0.0029592514038085938, "learning_rate": 1.4412211012432213e-05, "loss": -0.0099, "num_tokens": 16385736.0, "reward": 0.9362239241600037, "reward_std": 0.23804768919944763, "rewards/cross_solution_unittest_reward/mean": 0.3656328096985817, "rewards/cross_solution_unittest_reward/std": 0.28275594860315323, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 40 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 2157.0, "completions/max_terminated_length": 2157.0, "completions/mean_length": 504.46356201171875, "completions/mean_terminated_length": 504.46356201171875, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "entropy": 0.1143646240234375, "epoch": 0.3741089250071286, "frac_reward_zero_std": 0.0, "grad_norm": 0.032618340104818344, "kl": 0.0024976730346679688, "learning_rate": 1.4154150130018867e-05, "loss": -0.0, "num_tokens": 16799306.0, "reward": 0.8250623941421509, "reward_std": 0.2232421487569809, "rewards/cross_solution_unittest_reward/mean": 0.3419858068227768, "rewards/cross_solution_unittest_reward/std": 0.2102661430835724, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 41 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 875.0, "completions/max_terminated_length": 875.0, "completions/mean_length": 505.9583435058594, "completions/mean_terminated_length": 505.9583435058594, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.095428466796875, "epoch": 0.38323353293413176, "frac_reward_zero_std": 0.0, "grad_norm": 0.03039792738854885, "kl": 0.0020034313201904297, "learning_rate": 1.3892701063173917e-05, "loss": 0.0116, "num_tokens": 17244018.0, "reward": 1.0598524808883667, "reward_std": 0.22255584597587585, "rewards/cross_solution_unittest_reward/mean": 0.4190962612628937, "rewards/cross_solution_unittest_reward/std": 0.24228056520223618, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 42 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 744.0, "completions/mean_length": 487.9270935058594, "completions/mean_terminated_length": 479.8433532714844, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.1062774658203125, "epoch": 0.39235814086113485, "frac_reward_zero_std": 0.0, "grad_norm": 0.03161928430199623, "kl": 0.0024061203002929688, "learning_rate": 1.362807705350641e-05, "loss": -0.0093, "num_tokens": 17585734.0, "reward": 1.223128318786621, "reward_std": 0.23245906829833984, "rewards/cross_solution_unittest_reward/mean": 0.47970961034297943, "rewards/cross_solution_unittest_reward/std": 0.24992112070322037, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 43 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 1550.0, "completions/mean_length": 522.5390625, "completions/mean_terminated_length": 506.5104675292969, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.0976715087890625, "epoch": 0.401482748788138, "frac_reward_zero_std": 0.0, "grad_norm": 0.029316462576389313, "kl": 0.0023124217987060547, "learning_rate": 1.3360493932154301e-05, "loss": 0.0087, "num_tokens": 17997293.0, "reward": 0.770310640335083, "reward_std": 0.22640717029571533, "rewards/cross_solution_unittest_reward/mean": 0.3347120136022568, "rewards/cross_solution_unittest_reward/std": 0.2088947519659996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 44 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 761.0, "completions/max_terminated_length": 761.0, "completions/mean_length": 452.7578125, "completions/mean_terminated_length": 452.7578125, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.117156982421875, "epoch": 0.41060735671514115, "frac_reward_zero_std": 0.0, "grad_norm": 0.03837798163294792, "kl": 0.003719329833984375, "learning_rate": 1.3090169943749475e-05, "loss": 0.008, "num_tokens": 18373704.0, "reward": 1.1445835828781128, "reward_std": 0.27869975566864014, "rewards/cross_solution_unittest_reward/mean": 0.4463556706905365, "rewards/cross_solution_unittest_reward/std": 0.2553638890385628, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 45 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 950.0, "completions/max_terminated_length": 950.0, "completions/mean_length": 496.21875, "completions/mean_terminated_length": 496.21875, "completions/min_length": 258.0, "completions/min_terminated_length": 258.0, "entropy": 0.10125732421875, "epoch": 0.4197319646421443, "frac_reward_zero_std": 0.0, "grad_norm": 0.032201867550611496, "kl": 0.002789020538330078, "learning_rate": 1.2817325568414299e-05, "loss": 0.0053, "num_tokens": 18784484.0, "reward": 1.0264325141906738, "reward_std": 0.3673638701438904, "rewards/cross_solution_unittest_reward/mean": 0.40224773436784744, "rewards/cross_solution_unittest_reward/std": 0.2333994135260582, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 46 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 1553.0, "completions/mean_length": 507.47137451171875, "completions/mean_terminated_length": 491.3638916015625, "completions/min_length": 252.0, "completions/min_terminated_length": 252.0, "entropy": 0.1202850341796875, "epoch": 0.42885657256914744, "frac_reward_zero_std": 0.0, "grad_norm": 0.032994262874126434, "kl": 0.0027620792388916016, "learning_rate": 1.2542183341934873e-05, "loss": 0.0142, "num_tokens": 19204529.0, "reward": 0.8056275844573975, "reward_std": 0.2351686954498291, "rewards/cross_solution_unittest_reward/mean": 0.32472602277994156, "rewards/cross_solution_unittest_reward/std": 0.2546079531311989, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 47 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 893.0, "completions/max_terminated_length": 893.0, "completions/mean_length": 490.56512451171875, "completions/mean_terminated_length": 490.56512451171875, "completions/min_length": 264.0, "completions/min_terminated_length": 264.0, "entropy": 0.107147216796875, "epoch": 0.43798118049615054, "frac_reward_zero_std": 0.0, "grad_norm": 0.031603384763002396, "kl": 0.0028133392333984375, "learning_rate": 1.2264967674257647e-05, "loss": 0.0043, "num_tokens": 19594874.0, "reward": 1.1109874248504639, "reward_std": 0.3147701919078827, "rewards/cross_solution_unittest_reward/mean": 0.4285571426153183, "rewards/cross_solution_unittest_reward/std": 0.2294345498085022, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 48 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1107.0, "completions/mean_length": 502.71356201171875, "completions/mean_terminated_length": 494.6684265136719, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "entropy": 0.100433349609375, "epoch": 0.4471057884231537, "frac_reward_zero_std": 0.0, "grad_norm": 0.0326952300965786, "kl": 0.0027174949645996094, "learning_rate": 1.1985904666457455e-05, "loss": 0.01, "num_tokens": 20009764.0, "reward": 0.8557624816894531, "reward_std": 0.3132660984992981, "rewards/cross_solution_unittest_reward/mean": 0.32815027981996536, "rewards/cross_solution_unittest_reward/std": 0.2442621961236, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 49 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1427.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 507.59375, "completions/mean_terminated_length": 507.59375, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "entropy": 0.0999755859375, "epoch": 0.45623039635015683, "frac_reward_zero_std": 0.0833333358168602, "grad_norm": 0.03101285733282566, "kl": 0.004076242446899414, "learning_rate": 1.170522192632624e-05, "loss": -0.004, "num_tokens": 20420760.0, "reward": 1.0653202533721924, "reward_std": 0.3440527319908142, "rewards/cross_solution_unittest_reward/mean": 0.41321125626564026, "rewards/cross_solution_unittest_reward/std": 0.28402765840291977, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 50 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1267.0, "completions/max_terminated_length": 1267.0, "completions/mean_length": 507.9349060058594, "completions/mean_terminated_length": 507.9349060058594, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.098052978515625, "epoch": 0.46535500427716, "frac_reward_zero_std": 0.0, "grad_norm": 0.03185436502099037, "kl": 0.003798246383666992, "learning_rate": 1.1423148382732854e-05, "loss": 0.0012, "num_tokens": 20840495.0, "reward": 1.1997592449188232, "reward_std": 0.2847609221935272, "rewards/cross_solution_unittest_reward/mean": 0.4686891883611679, "rewards/cross_solution_unittest_reward/std": 0.19803649932146072, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 51 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 920.0, "completions/mean_length": 490.5989685058594, "completions/mean_terminated_length": 482.5221862792969, "completions/min_length": 215.0, "completions/min_terminated_length": 215.0, "entropy": 0.1027984619140625, "epoch": 0.4744796122041631, "frac_reward_zero_std": 0.0, "grad_norm": 0.04125326871871948, "kl": 0.004448890686035156, "learning_rate": 1.1139914098905406e-05, "loss": 0.0064, "num_tokens": 21235525.0, "reward": 1.196807861328125, "reward_std": 0.3041377067565918, "rewards/cross_solution_unittest_reward/mean": 0.4725197106599808, "rewards/cross_solution_unittest_reward/std": 0.2374914139509201, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 52 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 836.0, "completions/max_terminated_length": 836.0, "completions/mean_length": 490.2421875, "completions/mean_terminated_length": 490.2421875, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.0958709716796875, "epoch": 0.4836042201311662, "frac_reward_zero_std": 0.0, "grad_norm": 0.0320790559053421, "kl": 0.002828359603881836, "learning_rate": 1.08557500847884e-05, "loss": 0.0042, "num_tokens": 21637346.0, "reward": 0.9321694374084473, "reward_std": 0.2492300122976303, "rewards/cross_solution_unittest_reward/mean": 0.3783368095755577, "rewards/cross_solution_unittest_reward/std": 0.2201835662126541, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 53 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 994.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 538.9453125, "completions/mean_terminated_length": 538.9453125, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "entropy": 0.103179931640625, "epoch": 0.49272882805816937, "frac_reward_zero_std": 0.0, "grad_norm": 0.03167517110705376, "kl": 0.0037932395935058594, "learning_rate": 1.0570888108627682e-05, "loss": 0.0041, "num_tokens": 22100661.0, "reward": 0.9747004508972168, "reward_std": 0.33161452412605286, "rewards/cross_solution_unittest_reward/mean": 0.37912487238645554, "rewards/cross_solution_unittest_reward/std": 0.20974859595298767, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 54 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 895.0, "completions/max_terminated_length": 895.0, "completions/mean_length": 479.8125, "completions/mean_terminated_length": 479.8125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.106231689453125, "epoch": 0.5018534359851725, "frac_reward_zero_std": 0.0, "grad_norm": 0.03148855268955231, "kl": 0.003476381301879883, "learning_rate": 1.0285560507936962e-05, "loss": 0.0042, "num_tokens": 22473749.0, "reward": 0.9761655330657959, "reward_std": 0.2898656129837036, "rewards/cross_solution_unittest_reward/mean": 0.37648654729127884, "rewards/cross_solution_unittest_reward/std": 0.21749861538410187, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 55 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 952.0, "completions/max_terminated_length": 952.0, "completions/mean_length": 487.0442810058594, "completions/mean_terminated_length": 487.0442810058594, "completions/min_length": 230.0, "completions/min_terminated_length": 230.0, "entropy": 0.1035308837890625, "epoch": 0.5109780439121756, "frac_reward_zero_std": 0.0, "grad_norm": 0.05067846551537514, "kl": 0.005573272705078125, "learning_rate": 1e-05, "loss": 0.0098, "num_tokens": 22845742.0, "reward": 1.0181424617767334, "reward_std": 0.238634392619133, "rewards/cross_solution_unittest_reward/mean": 0.4077446535229683, "rewards/cross_solution_unittest_reward/std": 0.21635101735591888, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 56 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 824.0, "completions/mean_length": 498.7552185058594, "completions/mean_terminated_length": 490.6997375488281, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "entropy": 0.104248046875, "epoch": 0.5201026518391788, "frac_reward_zero_std": 0.0, "grad_norm": 0.033629678189754486, "kl": 0.0038862228393554688, "learning_rate": 9.71443949206304e-06, "loss": 0.0042, "num_tokens": 23289888.0, "reward": 1.072588324546814, "reward_std": 0.23051133751869202, "rewards/cross_solution_unittest_reward/mean": 0.4333711564540863, "rewards/cross_solution_unittest_reward/std": 0.21058296412229538, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 57 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 1087.0, "completions/mean_length": 494.6536560058594, "completions/mean_terminated_length": 478.47906494140625, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "entropy": 0.1170196533203125, "epoch": 0.5292272597661819, "frac_reward_zero_std": 0.0, "grad_norm": 0.03605455532670021, "kl": 0.004413604736328125, "learning_rate": 9.42911189137232e-06, "loss": -0.0017, "num_tokens": 23705051.0, "reward": 1.092246413230896, "reward_std": 0.2363906353712082, "rewards/cross_solution_unittest_reward/mean": 0.42197316884994507, "rewards/cross_solution_unittest_reward/std": 0.2724035382270813, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 58 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 972.0, "completions/mean_length": 477.6224060058594, "completions/mean_terminated_length": 469.5117492675781, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "entropy": 0.11468505859375, "epoch": 0.5383518676931851, "frac_reward_zero_std": 0.0, "grad_norm": 0.035716503858566284, "kl": 0.004600048065185547, "learning_rate": 9.144249915211605e-06, "loss": 0.0055, "num_tokens": 24109890.0, "reward": 1.119308352470398, "reward_std": 0.28101611137390137, "rewards/cross_solution_unittest_reward/mean": 0.4181726723909378, "rewards/cross_solution_unittest_reward/std": 0.25985223799943924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 59 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 785.0, "completions/max_terminated_length": 785.0, "completions/mean_length": 492.97918701171875, "completions/mean_terminated_length": 492.97918701171875, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.09771728515625, "epoch": 0.5474764756201882, "frac_reward_zero_std": 0.0, "grad_norm": 0.0333021804690361, "kl": 0.004182338714599609, "learning_rate": 8.860085901094595e-06, "loss": -0.0062, "num_tokens": 24543226.0, "reward": 1.0735642910003662, "reward_std": 0.347337543964386, "rewards/cross_solution_unittest_reward/mean": 0.4236568510532379, "rewards/cross_solution_unittest_reward/std": 0.22879399359226227, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 60 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 943.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 529.3385620117188, "completions/mean_terminated_length": 529.3385620117188, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "entropy": 0.097503662109375, "epoch": 0.5566010835471913, "frac_reward_zero_std": 0.0, "grad_norm": 0.03290291503071785, "kl": 0.003975391387939453, "learning_rate": 8.576851617267151e-06, "loss": -0.0023, "num_tokens": 24991796.0, "reward": 1.1211392879486084, "reward_std": 0.28648895025253296, "rewards/cross_solution_unittest_reward/mean": 0.4426130950450897, "rewards/cross_solution_unittest_reward/std": 0.26173052191734314, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 61 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1262.0, "completions/mean_length": 533.1823120117188, "completions/mean_terminated_length": 525.2167358398438, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "entropy": 0.1041717529296875, "epoch": 0.5657256914741945, "frac_reward_zero_std": 0.0, "grad_norm": 0.032658372074365616, "kl": 0.0042781829833984375, "learning_rate": 8.294778073673762e-06, "loss": 0.0154, "num_tokens": 25421578.0, "reward": 0.8346015810966492, "reward_std": 0.2451450526714325, "rewards/cross_solution_unittest_reward/mean": 0.3612000532448292, "rewards/cross_solution_unittest_reward/std": 0.18463484197854996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 62 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 943.0, "completions/max_terminated_length": 943.0, "completions/mean_length": 465.7630310058594, "completions/mean_terminated_length": 465.7630310058594, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.1036376953125, "epoch": 0.5748502994011976, "frac_reward_zero_std": 0.0, "grad_norm": 0.05787957087159157, "kl": 0.005153656005859375, "learning_rate": 8.014095333542548e-06, "loss": 0.0039, "num_tokens": 25789535.0, "reward": 0.9333815574645996, "reward_std": 0.259033203125, "rewards/cross_solution_unittest_reward/mean": 0.37771864980459213, "rewards/cross_solution_unittest_reward/std": 0.2220945730805397, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 63 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 831.0, "completions/max_terminated_length": 831.0, "completions/mean_length": 480.3515625, "completions/mean_terminated_length": 480.3515625, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "entropy": 0.10076904296875, "epoch": 0.5839749073282008, "frac_reward_zero_std": 0.0, "grad_norm": 0.03183058649301529, "kl": 0.0033092498779296875, "learning_rate": 7.735032325742355e-06, "loss": 0.0006, "num_tokens": 26177366.0, "reward": 1.4325106143951416, "reward_std": 0.3147215247154236, "rewards/cross_solution_unittest_reward/mean": 0.535786435008049, "rewards/cross_solution_unittest_reward/std": 0.19719675183296204, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 64 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 921.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 476.0442810058594, "completions/mean_terminated_length": 476.0442810058594, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "entropy": 0.1140289306640625, "epoch": 0.5930995152552039, "frac_reward_zero_std": 0.0, "grad_norm": 0.036758169531822205, "kl": 0.005259037017822266, "learning_rate": 7.4578166580651335e-06, "loss": -0.0075, "num_tokens": 26565743.0, "reward": 1.1525869369506836, "reward_std": 0.29967427253723145, "rewards/cross_solution_unittest_reward/mean": 0.44647954404354095, "rewards/cross_solution_unittest_reward/std": 0.30218902230262756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 65 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.9375, "completions/max_length": 3584.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 543.9818115234375, "completions/mean_terminated_length": 511.9815979003906, "completions/min_length": 229.0, "completions/min_terminated_length": 229.0, "entropy": 0.1009979248046875, "epoch": 0.602224123182207, "frac_reward_zero_std": 0.0, "grad_norm": 0.03079250641167164, "kl": 0.00479888916015625, "learning_rate": 7.182674431585703e-06, "loss": 0.0201, "num_tokens": 27046624.0, "reward": 1.330259084701538, "reward_std": 0.28341954946517944, "rewards/cross_solution_unittest_reward/mean": 0.4929102957248688, "rewards/cross_solution_unittest_reward/std": 0.2743557021021843, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 66 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1038.0, "completions/max_terminated_length": 1038.0, "completions/mean_length": 488.4270935058594, "completions/mean_terminated_length": 488.4270935058594, "completions/min_length": 243.0, "completions/min_terminated_length": 243.0, "entropy": 0.096893310546875, "epoch": 0.6113487311092102, "frac_reward_zero_std": 0.0, "grad_norm": 0.034310225397348404, "kl": 0.0051801204681396484, "learning_rate": 6.909830056250527e-06, "loss": 0.0019, "num_tokens": 27499308.0, "reward": 1.3934414386749268, "reward_std": 0.35809773206710815, "rewards/cross_solution_unittest_reward/mean": 0.5251588076353073, "rewards/cross_solution_unittest_reward/std": 0.24779481440782547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 67 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 889.0, "completions/max_terminated_length": 889.0, "completions/mean_length": 519.8125, "completions/mean_terminated_length": 519.8125, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "entropy": 0.0970611572265625, "epoch": 0.6204733390362133, "frac_reward_zero_std": 0.0, "grad_norm": 0.03318794071674347, "kl": 0.003513336181640625, "learning_rate": 6.639506067845698e-06, "loss": -0.0132, "num_tokens": 27901196.0, "reward": 1.1413233280181885, "reward_std": 0.27733054757118225, "rewards/cross_solution_unittest_reward/mean": 0.4484158530831337, "rewards/cross_solution_unittest_reward/std": 0.20082928240299225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 68 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 879.0, "completions/max_terminated_length": 879.0, "completions/mean_length": 482.9661560058594, "completions/mean_terminated_length": 482.9661560058594, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.1024017333984375, "epoch": 0.6295979469632165, "frac_reward_zero_std": 0.0, "grad_norm": 0.041993044316768646, "kl": 0.006025791168212891, "learning_rate": 6.3719229464935915e-06, "loss": -0.0028, "num_tokens": 28332687.0, "reward": 1.1847600936889648, "reward_std": 0.2705700099468231, "rewards/cross_solution_unittest_reward/mean": 0.46874039620161057, "rewards/cross_solution_unittest_reward/std": 0.20285168290138245, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 69 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1062.0, "completions/mean_length": 532.9896240234375, "completions/mean_terminated_length": 525.0234985351562, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "entropy": 0.0945587158203125, "epoch": 0.6387225548902196, "frac_reward_zero_std": 0.0, "grad_norm": 0.0388309545814991, "kl": 0.004973173141479492, "learning_rate": 6.107298936826086e-06, "loss": -0.0019, "num_tokens": 28783291.0, "reward": 1.2814630270004272, "reward_std": 0.39242124557495117, "rewards/cross_solution_unittest_reward/mean": 0.4748888313770294, "rewards/cross_solution_unittest_reward/std": 0.21190231293439865, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 70 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 955.0, "completions/max_terminated_length": 955.0, "completions/mean_length": 495.1302185058594, "completions/mean_terminated_length": 495.1302185058594, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.1051025390625, "epoch": 0.6478471628172227, "frac_reward_zero_std": 0.0, "grad_norm": 0.03422863036394119, "kl": 0.003699779510498047, "learning_rate": 5.845849869981137e-06, "loss": -0.0108, "num_tokens": 29185853.0, "reward": 0.7699536085128784, "reward_std": 0.227044939994812, "rewards/cross_solution_unittest_reward/mean": 0.33149589970707893, "rewards/cross_solution_unittest_reward/std": 0.19518966227769852, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 71 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1106.0, "completions/mean_length": 521.8333740234375, "completions/mean_terminated_length": 513.838134765625, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "entropy": 0.1019439697265625, "epoch": 0.6569717707442259, "frac_reward_zero_std": 0.0, "grad_norm": 0.1640857458114624, "kl": 0.015253067016601562, "learning_rate": 5.587788987567785e-06, "loss": -0.0029, "num_tokens": 29597149.0, "reward": 1.2916417121887207, "reward_std": 0.2836645245552063, "rewards/cross_solution_unittest_reward/mean": 0.49249671399593353, "rewards/cross_solution_unittest_reward/std": 0.2682873010635376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 72 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 772.0, "completions/mean_length": 483.203125, "completions/mean_terminated_length": 475.1070556640625, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.1068267822265625, "epoch": 0.666096378671229, "frac_reward_zero_std": 0.0, "grad_norm": 0.03074985183775425, "kl": 0.003566741943359375, "learning_rate": 5.333326767743263e-06, "loss": 0.0117, "num_tokens": 30000275.0, "reward": 1.3340082168579102, "reward_std": 0.30908769369125366, "rewards/cross_solution_unittest_reward/mean": 0.5031353682279587, "rewards/cross_solution_unittest_reward/std": 0.23924725502729416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 73 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1477.0, "completions/max_terminated_length": 1477.0, "completions/mean_length": 533.8151245117188, "completions/mean_terminated_length": 533.8151245117188, "completions/min_length": 211.0, "completions/min_terminated_length": 211.0, "entropy": 0.0978546142578125, "epoch": 0.6752209865982322, "frac_reward_zero_std": 0.0, "grad_norm": 0.029303815215826035, "kl": 0.003360271453857422, "learning_rate": 5.082670753543961e-06, "loss": -0.0034, "num_tokens": 30411700.0, "reward": 1.1401095390319824, "reward_std": 0.32691219449043274, "rewards/cross_solution_unittest_reward/mean": 0.4419480115175247, "rewards/cross_solution_unittest_reward/std": 0.24548739194869995, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 74 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1453.0, "completions/max_terminated_length": 1453.0, "completions/mean_length": 520.4713745117188, "completions/mean_terminated_length": 520.4713745117188, "completions/min_length": 268.0, "completions/min_terminated_length": 268.0, "entropy": 0.109771728515625, "epoch": 0.6843455945252352, "frac_reward_zero_std": 0.0, "grad_norm": 0.03002401813864708, "kl": 0.003072500228881836, "learning_rate": 4.836025383610382e-06, "loss": 0.0128, "num_tokens": 30793049.0, "reward": 1.1908866167068481, "reward_std": 0.269525408744812, "rewards/cross_solution_unittest_reward/mean": 0.46111442148685455, "rewards/cross_solution_unittest_reward/std": 0.23126034438610077, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 75 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1016.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 458.9505310058594, "completions/mean_terminated_length": 458.9505310058594, "completions/min_length": 241.0, "completions/min_terminated_length": 241.0, "entropy": 0.1105194091796875, "epoch": 0.6934702024522383, "frac_reward_zero_std": 0.0, "grad_norm": 0.033196672797203064, "kl": 0.003894329071044922, "learning_rate": 4.593591825444028e-06, "loss": 0.0082, "num_tokens": 31135174.0, "reward": 0.9832911491394043, "reward_std": 0.23262672126293182, "rewards/cross_solution_unittest_reward/mean": 0.3907434269785881, "rewards/cross_solution_unittest_reward/std": 0.22558597475290298, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 76 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 499.5442810058594, "completions/mean_terminated_length": 491.4908752441406, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "entropy": 0.1078948974609375, "epoch": 0.7025948103792415, "frac_reward_zero_std": 0.0, "grad_norm": 0.033747926354408264, "kl": 0.003387928009033203, "learning_rate": 4.355567811332311e-06, "loss": 0.0017, "num_tokens": 31541479.0, "reward": 1.2079272270202637, "reward_std": 0.29688721895217896, "rewards/cross_solution_unittest_reward/mean": 0.4636748433113098, "rewards/cross_solution_unittest_reward/std": 0.2018030509352684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 77 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 3241.0, "completions/max_terminated_length": 3241.0, "completions/mean_length": 514.5677490234375, "completions/mean_terminated_length": 514.5677490234375, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.0965728759765625, "epoch": 0.7117194183062446, "frac_reward_zero_std": 0.0, "grad_norm": 0.03385668992996216, "kl": 0.0037603378295898438, "learning_rate": 4.12214747707527e-06, "loss": 0.0088, "num_tokens": 31984281.0, "reward": 1.2030308246612549, "reward_std": 0.3671332597732544, "rewards/cross_solution_unittest_reward/mean": 0.46230846643447876, "rewards/cross_solution_unittest_reward/std": 0.2557263821363449, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 78 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, "completions/max_terminated_length": 3389.0, "completions/mean_length": 526.1927490234375, "completions/mean_terminated_length": 502.115478515625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.1031036376953125, "epoch": 0.7208440262332478, "frac_reward_zero_std": 0.0, "grad_norm": 0.0314985066652298, "kl": 0.003326892852783203, "learning_rate": 3.893521203645618e-06, "loss": 0.0163, "num_tokens": 32385115.0, "reward": 1.1673388481140137, "reward_std": 0.2541990876197815, "rewards/cross_solution_unittest_reward/mean": 0.4651763513684273, "rewards/cross_solution_unittest_reward/std": 0.22935150563716888, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 79 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 837.0, "completions/mean_length": 480.5520935058594, "completions/mean_terminated_length": 472.4490966796875, "completions/min_length": 236.0, "completions/min_terminated_length": 236.0, "entropy": 0.100311279296875, "epoch": 0.7299686341602509, "frac_reward_zero_std": 0.0, "grad_norm": 0.03507440537214279, "kl": 0.004328250885009766, "learning_rate": 3.6698754619112974e-06, "loss": -0.0013, "num_tokens": 32748231.0, "reward": 1.034367561340332, "reward_std": 0.23578712344169617, "rewards/cross_solution_unittest_reward/mean": 0.4301436319947243, "rewards/cross_solution_unittest_reward/std": 0.23155031353235245, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 80 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 898.0, "completions/max_terminated_length": 898.0, "completions/mean_length": 479.953125, "completions/mean_terminated_length": 479.953125, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.1009063720703125, "epoch": 0.739093242087254, "frac_reward_zero_std": 0.0, "grad_norm": 0.030309055000543594, "kl": 0.0035872459411621094, "learning_rate": 3.4513926605471504e-06, "loss": -0.0042, "num_tokens": 33129445.0, "reward": 1.2114933729171753, "reward_std": 0.22488722205162048, "rewards/cross_solution_unittest_reward/mean": 0.4736487567424774, "rewards/cross_solution_unittest_reward/std": 0.20029103010892868, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 81 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1235.0, "completions/max_terminated_length": 1235.0, "completions/mean_length": 479.26824951171875, "completions/mean_terminated_length": 479.26824951171875, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "entropy": 0.105133056640625, "epoch": 0.7482178500142572, "frac_reward_zero_std": 0.0, "grad_norm": 0.05099751427769661, "kl": 0.007653713226318359, "learning_rate": 3.2382509972598087e-06, "loss": -0.0018, "num_tokens": 33519820.0, "reward": 1.122490406036377, "reward_std": 0.26781564950942993, "rewards/cross_solution_unittest_reward/mean": 0.4443480148911476, "rewards/cross_solution_unittest_reward/std": 0.24169741570949554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 82 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 901.0, "completions/mean_length": 487.75262451171875, "completions/mean_terminated_length": 479.6684265136719, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.104339599609375, "epoch": 0.7573424579412603, "frac_reward_zero_std": 0.0, "grad_norm": 0.035204458981752396, "kl": 0.004299163818359375, "learning_rate": 3.0306243134470668e-06, "loss": 0.0056, "num_tokens": 33916373.0, "reward": 1.06869375705719, "reward_std": 0.3348367214202881, "rewards/cross_solution_unittest_reward/mean": 0.4094615653157234, "rewards/cross_solution_unittest_reward/std": 0.23972471803426743, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 83 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.890625, "completions/max_length": 3584.0, "completions/max_terminated_length": 767.0, "completions/mean_length": 528.9635620117188, "completions/mean_terminated_length": 472.23870849609375, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "entropy": 0.10943603515625, "epoch": 0.7664670658682635, "frac_reward_zero_std": 0.0, "grad_norm": 0.03322676569223404, "kl": 0.0032072067260742188, "learning_rate": 2.8286819524103657e-06, "loss": 0.0142, "num_tokens": 34275463.0, "reward": 1.1465957164764404, "reward_std": 0.2977037727832794, "rewards/cross_solution_unittest_reward/mean": 0.44328781962394714, "rewards/cross_solution_unittest_reward/std": 0.22255534678697586, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 84 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 2205.0, "completions/mean_length": 511.40625, "completions/mean_terminated_length": 503.3838195800781, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "entropy": 0.098388671875, "epoch": 0.7755916737952666, "frac_reward_zero_std": 0.0, "grad_norm": 0.04618420824408531, "kl": 0.00617527961730957, "learning_rate": 2.6325886212359496e-06, "loss": 0.0117, "num_tokens": 34711467.0, "reward": 1.1300448179244995, "reward_std": 0.23039524257183075, "rewards/cross_solution_unittest_reward/mean": 0.4431014582514763, "rewards/cross_solution_unittest_reward/std": 0.19984686374664307, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 85 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1019.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 476.16668701171875, "completions/mean_terminated_length": 476.16668701171875, "completions/min_length": 182.0, "completions/min_terminated_length": 182.0, "entropy": 0.1077423095703125, "epoch": 0.7847162817222697, "frac_reward_zero_std": 0.0, "grad_norm": 0.0404982790350914, "kl": 0.004585742950439453, "learning_rate": 2.4425042564574186e-06, "loss": 0.0068, "num_tokens": 35123059.0, "reward": 1.0940883159637451, "reward_std": 0.3068244159221649, "rewards/cross_solution_unittest_reward/mean": 0.432598315179348, "rewards/cross_solution_unittest_reward/std": 0.20239350944757462, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 86 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.953125, "completions/max_length": 3584.0, "completions/max_terminated_length": 2127.0, "completions/mean_length": 533.90625, "completions/mean_terminated_length": 509.8897705078125, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.1087188720703125, "epoch": 0.7938408896492729, "frac_reward_zero_std": 0.0, "grad_norm": 0.0433526337146759, "kl": 0.004700660705566406, "learning_rate": 2.2585838936091753e-06, "loss": 0.0154, "num_tokens": 35560487.0, "reward": 1.2438384294509888, "reward_std": 0.2887391448020935, "rewards/cross_solution_unittest_reward/mean": 0.4800245463848114, "rewards/cross_solution_unittest_reward/std": 0.22094998508691788, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 87 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 803.0, "completions/mean_length": 444.28387451171875, "completions/mean_terminated_length": 436.086181640625, "completions/min_length": 216.0, "completions/min_terminated_length": 216.0, "entropy": 0.111419677734375, "epoch": 0.802965497576276, "frac_reward_zero_std": 0.0, "grad_norm": 0.03479230776429176, "kl": 0.003818035125732422, "learning_rate": 2.0809775407772505e-06, "loss": -0.003, "num_tokens": 35918492.0, "reward": 1.212603211402893, "reward_std": 0.3318028450012207, "rewards/cross_solution_unittest_reward/mean": 0.44981423020362854, "rewards/cross_solution_unittest_reward/std": 0.22631778568029404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 88 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 880.0, "completions/max_terminated_length": 880.0, "completions/mean_length": 510.78125, "completions/mean_terminated_length": 510.78125, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "entropy": 0.0924224853515625, "epoch": 0.8120901055032792, "frac_reward_zero_std": 0.0, "grad_norm": 0.030438624322414398, "kl": 0.0034804344177246094, "learning_rate": 1.9098300562505266e-06, "loss": 0.0079, "num_tokens": 36344032.0, "reward": 1.1114070415496826, "reward_std": 0.2217615246772766, "rewards/cross_solution_unittest_reward/mean": 0.4325195252895355, "rewards/cross_solution_unittest_reward/std": 0.2371705397963524, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 89 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 875.0, "completions/max_terminated_length": 875.0, "completions/mean_length": 486.8203125, "completions/mean_terminated_length": 486.8203125, "completions/min_length": 248.0, "completions/min_terminated_length": 248.0, "entropy": 0.0999908447265625, "epoch": 0.8212147134302823, "frac_reward_zero_std": 0.0, "grad_norm": 0.040998078882694244, "kl": 0.008658885955810547, "learning_rate": 1.74528103037226e-06, "loss": 0.006, "num_tokens": 36757971.0, "reward": 1.045344591140747, "reward_std": 0.34945201873779297, "rewards/cross_solution_unittest_reward/mean": 0.3958388417959213, "rewards/cross_solution_unittest_reward/std": 0.27266910672187805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 90 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1187.0, "completions/max_terminated_length": 1187.0, "completions/mean_length": 489.68231201171875, "completions/mean_terminated_length": 489.68231201171875, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.105194091796875, "epoch": 0.8303393213572854, "frac_reward_zero_std": 0.0, "grad_norm": 0.03380502387881279, "kl": 0.003403186798095703, "learning_rate": 1.587464671688187e-06, "loss": 0.0057, "num_tokens": 37152673.0, "reward": 1.0752450227737427, "reward_std": 0.20711544156074524, "rewards/cross_solution_unittest_reward/mean": 0.4331958070397377, "rewards/cross_solution_unittest_reward/std": 0.20185105502605438, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 91 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 1024.0, "completions/mean_length": 494.28387451171875, "completions/mean_terminated_length": 494.28387451171875, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "entropy": 0.1022796630859375, "epoch": 0.8394639292842886, "frac_reward_zero_std": 0.0, "grad_norm": 0.030283644795417786, "kl": 0.0035581588745117188, "learning_rate": 1.436509697484111e-06, "loss": -0.0023, "num_tokens": 37567030.0, "reward": 1.0902509689331055, "reward_std": 0.27206262946128845, "rewards/cross_solution_unittest_reward/mean": 0.42259418964385986, "rewards/cross_solution_unittest_reward/std": 0.244268037378788, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 92 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.96875, "completions/max_length": 3584.0, "completions/max_terminated_length": 1057.0, "completions/mean_length": 564.1666870117188, "completions/mean_terminated_length": 548.3560180664062, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "entropy": 0.0888824462890625, "epoch": 0.8485885372112917, "frac_reward_zero_std": 0.0, "grad_norm": 1.1316825151443481, "kl": 0.043515682220458984, "learning_rate": 1.2925392288022299e-06, "loss": 0.0208, "num_tokens": 38002926.0, "reward": 1.0368365049362183, "reward_std": 0.2775520086288452, "rewards/cross_solution_unittest_reward/mean": 0.4123196601867676, "rewards/cross_solution_unittest_reward/std": 0.22595145553350449, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 93 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 973.0, "completions/mean_length": 524.9296875, "completions/mean_terminated_length": 516.9425659179688, "completions/min_length": 227.0, "completions/min_terminated_length": 227.0, "entropy": 0.0940704345703125, "epoch": 0.8577131451382949, "frac_reward_zero_std": 0.0, "grad_norm": 0.028070101514458656, "kl": 0.0034241676330566406, "learning_rate": 1.1556706900218572e-06, "loss": -0.0006, "num_tokens": 38420499.0, "reward": 1.074540615081787, "reward_std": 0.32584017515182495, "rewards/cross_solution_unittest_reward/mean": 0.41290927678346634, "rewards/cross_solution_unittest_reward/std": 0.2791058123111725, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 94 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1074.0, "completions/max_terminated_length": 1074.0, "completions/mean_length": 493.265625, "completions/mean_terminated_length": 493.265625, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "entropy": 0.09954833984375, "epoch": 0.866837753065298, "frac_reward_zero_std": 0.0, "grad_norm": 0.03178764134645462, "kl": 0.003660440444946289, "learning_rate": 1.0260157130864178e-06, "loss": 0.0012, "num_tokens": 38866593.0, "reward": 1.128114104270935, "reward_std": 0.27225562930107117, "rewards/cross_solution_unittest_reward/mean": 0.43701624870300293, "rewards/cross_solution_unittest_reward/std": 0.18694043159484863, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 95 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1080.0, "completions/max_terminated_length": 1080.0, "completions/mean_length": 523.0546875, "completions/mean_terminated_length": 523.0546875, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "entropy": 0.100311279296875, "epoch": 0.8759623609923011, "frac_reward_zero_std": 0.0, "grad_norm": 0.032469868659973145, "kl": 0.003574371337890625, "learning_rate": 9.036800464548157e-07, "loss": 0.0029, "num_tokens": 39255286.0, "reward": 1.0960664749145508, "reward_std": 0.25419503450393677, "rewards/cross_solution_unittest_reward/mean": 0.41909460723400116, "rewards/cross_solution_unittest_reward/std": 0.2185814157128334, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 96 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 793.0, "completions/mean_length": 463.96356201171875, "completions/mean_terminated_length": 455.8172302246094, "completions/min_length": 178.0, "completions/min_terminated_length": 178.0, "entropy": 0.1122283935546875, "epoch": 0.8850869689193043, "frac_reward_zero_std": 0.0, "grad_norm": 0.03675765171647072, "kl": 0.0036330223083496094, "learning_rate": 7.887634688515e-07, "loss": 0.0198, "num_tokens": 39578848.0, "reward": 1.2880136966705322, "reward_std": 0.2283666580915451, "rewards/cross_solution_unittest_reward/mean": 0.501230925321579, "rewards/cross_solution_unittest_reward/std": 0.22084086388349533, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 97 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 898.0, "completions/max_terminated_length": 898.0, "completions/mean_length": 530.3646240234375, "completions/mean_terminated_length": 530.3646240234375, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "entropy": 0.099212646484375, "epoch": 0.8942115768463074, "frac_reward_zero_std": 0.0, "grad_norm": 0.03102123737335205, "kl": 0.0036611557006835938, "learning_rate": 6.813597078854772e-07, "loss": -0.0129, "num_tokens": 40025780.0, "reward": 1.2561891078948975, "reward_std": 0.3024594485759735, "rewards/cross_solution_unittest_reward/mean": 0.48492124676704407, "rewards/cross_solution_unittest_reward/std": 0.2765001207590103, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 98 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 492.99481201171875, "completions/mean_terminated_length": 484.9242858886719, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.103118896484375, "epoch": 0.9033361847733106, "frac_reward_zero_std": 0.0, "grad_norm": 0.031520117074251175, "kl": 0.003575563430786133, "learning_rate": 5.815563636047539e-07, "loss": -0.0028, "num_tokens": 40397250.0, "reward": 1.2597386837005615, "reward_std": 0.2905423939228058, "rewards/cross_solution_unittest_reward/mean": 0.4819524586200714, "rewards/cross_solution_unittest_reward/std": 0.21879704296588898, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 99 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 928.0, "completions/max_terminated_length": 928.0, "completions/mean_length": 472.08856201171875, "completions/mean_terminated_length": 472.08856201171875, "completions/min_length": 201.0, "completions/min_terminated_length": 201.0, "entropy": 0.11407470703125, "epoch": 0.9124607927003137, "frac_reward_zero_std": 0.0, "grad_norm": 0.033804018050432205, "kl": 0.003757953643798828, "learning_rate": 4.894348370484648e-07, "loss": -0.0169, "num_tokens": 40820412.0, "reward": 1.1143543720245361, "reward_std": 0.2880322337150574, "rewards/cross_solution_unittest_reward/mean": 0.42812392115592957, "rewards/cross_solution_unittest_reward/std": 0.24535022675991058, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 100 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 891.0, "completions/max_terminated_length": 891.0, "completions/mean_length": 476.58074951171875, "completions/mean_terminated_length": 476.58074951171875, "completions/min_length": 262.0, "completions/min_terminated_length": 262.0, "entropy": 0.09490966796875, "epoch": 0.9215854006273168, "frac_reward_zero_std": 0.0, "grad_norm": 0.031989987939596176, "kl": 0.0035381317138671875, "learning_rate": 4.0507026385502747e-07, "loss": 0.0016, "num_tokens": 41205987.0, "reward": 1.1675355434417725, "reward_std": 0.27103692293167114, "rewards/cross_solution_unittest_reward/mean": 0.4633182883262634, "rewards/cross_solution_unittest_reward/std": 0.19777817279100418, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 101 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 886.0, "completions/mean_length": 509.75262451171875, "completions/mean_terminated_length": 501.7258605957031, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "entropy": 0.09368896484375, "epoch": 0.93071000855432, "frac_reward_zero_std": 0.0, "grad_norm": 0.06643658876419067, "kl": 0.004868507385253906, "learning_rate": 3.2853145298042954e-07, "loss": 0.0168, "num_tokens": 41637052.0, "reward": 1.360771894454956, "reward_std": 0.33854445815086365, "rewards/cross_solution_unittest_reward/mean": 0.5105735212564468, "rewards/cross_solution_unittest_reward/std": 0.19905748218297958, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 102 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 857.0, "completions/max_terminated_length": 857.0, "completions/mean_length": 468.79949951171875, "completions/mean_terminated_length": 468.79949951171875, "completions/min_length": 238.0, "completions/min_terminated_length": 238.0, "entropy": 0.1016998291015625, "epoch": 0.939834616481323, "frac_reward_zero_std": 0.0, "grad_norm": 0.044055838137865067, "kl": 0.003973722457885742, "learning_rate": 2.5988083057666534e-07, "loss": 0.0033, "num_tokens": 42007991.0, "reward": 1.3666280508041382, "reward_std": 0.3037351369857788, "rewards/cross_solution_unittest_reward/mean": 0.5158256590366364, "rewards/cross_solution_unittest_reward/std": 0.24741099774837494, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 103 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 921.0, "completions/max_terminated_length": 921.0, "completions/mean_length": 493.125, "completions/mean_terminated_length": 493.125, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "entropy": 0.09735107421875, "epoch": 0.9489592244083263, "frac_reward_zero_std": 0.0, "grad_norm": 0.03200039267539978, "kl": 0.004022121429443359, "learning_rate": 1.9917438907606556e-07, "loss": -0.0008, "num_tokens": 42423575.0, "reward": 1.1082267761230469, "reward_std": 0.2577505111694336, "rewards/cross_solution_unittest_reward/mean": 0.4455152079463005, "rewards/cross_solution_unittest_reward/std": 0.19009340554475784, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 104 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 3546.0, "completions/max_terminated_length": 3546.0, "completions/mean_length": 478.2317810058594, "completions/mean_terminated_length": 478.2317810058594, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.10748291015625, "epoch": 0.9580838323353293, "frac_reward_zero_std": 0.0, "grad_norm": 0.03472354635596275, "kl": 0.003666400909423828, "learning_rate": 1.464616415230702e-07, "loss": 0.0101, "num_tokens": 42791328.0, "reward": 0.8813101649284363, "reward_std": 0.30557072162628174, "rewards/cross_solution_unittest_reward/mean": 0.3544023931026459, "rewards/cross_solution_unittest_reward/std": 0.22855371981859207, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 105 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -4.984375, "completions/max_length": 3584.0, "completions/max_terminated_length": 1037.0, "completions/mean_length": 535.1146240234375, "completions/mean_terminated_length": 527.154052734375, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "entropy": 0.09112548828125, "epoch": 0.9672084402623324, "frac_reward_zero_std": 0.0, "grad_norm": 0.027567479759454727, "kl": 0.0032286643981933594, "learning_rate": 1.0178558119067316e-07, "loss": -0.0102, "num_tokens": 43222180.0, "reward": 1.1219594478607178, "reward_std": 0.34339237213134766, "rewards/cross_solution_unittest_reward/mean": 0.4241596907377243, "rewards/cross_solution_unittest_reward/std": 0.2556988596916199, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 106 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 840.0, "completions/max_terminated_length": 840.0, "completions/mean_length": 493.3828125, "completions/mean_terminated_length": 493.3828125, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "entropy": 0.112396240234375, "epoch": 0.9763330481893356, "frac_reward_zero_std": 0.0, "grad_norm": 0.036228280514478683, "kl": 0.004697322845458984, "learning_rate": 6.51826465144978e-08, "loss": -0.0014, "num_tokens": 43626183.0, "reward": 1.1601941585540771, "reward_std": 0.27455127239227295, "rewards/cross_solution_unittest_reward/mean": 0.457772821187973, "rewards/cross_solution_unittest_reward/std": 0.23431915044784546, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 107 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 1087.0, "completions/max_terminated_length": 1087.0, "completions/mean_length": 527.7291870117188, "completions/mean_terminated_length": 527.7291870117188, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "entropy": 0.0887908935546875, "epoch": 0.9854576561163387, "frac_reward_zero_std": 0.0, "grad_norm": 0.02786332555115223, "kl": 0.002989053726196289, "learning_rate": 3.668269137308666e-08, "loss": -0.005, "num_tokens": 44066847.0, "reward": 1.3689097166061401, "reward_std": 0.2395750731229782, "rewards/cross_solution_unittest_reward/mean": 0.5246002376079559, "rewards/cross_solution_unittest_reward/std": 0.2055259644985199, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 108 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": -5.0, "completions/max_length": 819.0, "completions/max_terminated_length": 819.0, "completions/mean_length": 473.96356201171875, "completions/mean_terminated_length": 473.96356201171875, "completions/min_length": 202.0, "completions/min_terminated_length": 202.0, "entropy": 0.1051177978515625, "epoch": 0.9945822640433419, "frac_reward_zero_std": 0.0, "grad_norm": 0.03451412171125412, "kl": 0.0034637451171875, "learning_rate": 1.630896073864352e-08, "loss": 0.0043, "num_tokens": 44441753.0, "reward": 1.357940435409546, "reward_std": 0.3025989532470703, "rewards/cross_solution_unittest_reward/mean": 0.5211868733167648, "rewards/cross_solution_unittest_reward/std": 0.2235267534852028, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "step": 109 }, { "epoch": 0.9945822640433419, "step": 109, "total_flos": 0.0, "train_loss": 0.002528898397298306, "train_runtime": 42447.4995, "train_samples_per_second": 0.031, "train_steps_per_second": 0.003 } ], "logging_steps": 1, "max_steps": 110, "num_input_tokens_seen": 44441753, "num_train_epochs": 1, "save_steps": 25, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": false }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 2, "trial_name": null, "trial_params": null }