diff --git "a/trainer_state.json" "b/trainer_state.json" new file mode 100644--- /dev/null +++ "b/trainer_state.json" @@ -0,0 +1,4033 @@ +{ + "best_metric": null, + "best_model_checkpoint": null, + "epoch": 0.03345376689415228, + "eval_steps": 500, + "global_step": 250, + "is_hyper_param_search": false, + "is_local_process_zero": true, + "is_world_process_zero": true, + "log_history": [ + { + "completion_length": 200.0, + "epoch": 0.00013381506757660912, + "grad_norm": 0.0, + "kl": 0.0, + "learning_rate": 2.0000000000000002e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 1 + }, + { + "completion_length": 168.6666717529297, + "epoch": 0.00026763013515321824, + "grad_norm": 0.6520445942878723, + "kl": 0.0, + "learning_rate": 4.0000000000000003e-07, + "loss": -0.0, + "reward": -0.046833336353302, + "reward_std": 0.1817607581615448, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.046833336353302, + "step": 2 + }, + { + "completion_length": 128.1666717529297, + "epoch": 0.0004014452027298274, + "grad_norm": 0.7553274631500244, + "kl": 0.00023205635079648346, + "learning_rate": 6.000000000000001e-07, + "loss": 0.0, + "reward": 0.7700001001358032, + "reward_std": 1.30967378616333, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.06333333253860474, + "step": 3 + }, + { + "completion_length": 181.83334350585938, + "epoch": 0.0005352602703064365, + "grad_norm": 0.0009925751946866512, + "kl": 0.0003016882692463696, + "learning_rate": 8.000000000000001e-07, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 4 + }, + { + "completion_length": 198.5, + "epoch": 0.0006690753378830456, + "grad_norm": 0.257866770029068, + "kl": 0.00034359516575932503, + "learning_rate": 1.0000000000000002e-06, + "loss": 0.0, + "reward": 0.4375, + "reward_std": 1.0116509199142456, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 5 + }, + { + "completion_length": 123.83333587646484, + "epoch": 0.0008028904054596548, + "grad_norm": 0.5083842277526855, + "kl": 0.00048388767754659057, + "learning_rate": 1.2000000000000002e-06, + "loss": 0.0, + "reward": -0.09333333373069763, + "reward_std": 0.112388014793396, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.09333333373069763, + "step": 6 + }, + { + "completion_length": 121.5, + "epoch": 0.0009367054730362638, + "grad_norm": 0.6899563670158386, + "kl": 0.0006710832240059972, + "learning_rate": 1.4000000000000001e-06, + "loss": 0.0, + "reward": 0.4321666955947876, + "reward_std": 1.0167949199676514, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.015500001609325409, + "step": 7 + }, + { + "completion_length": 155.1666717529297, + "epoch": 0.001070520540612873, + "grad_norm": 0.4054330587387085, + "kl": 0.0002861042448785156, + "learning_rate": 1.6000000000000001e-06, + "loss": 0.0, + "reward": 0.8665000200271606, + "reward_std": 1.2465369701385498, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.03316666558384895, + "step": 8 + }, + { + "completion_length": 146.6666717529297, + "epoch": 0.0012043356081894822, + "grad_norm": 0.3640693128108978, + "kl": 0.0003890148364007473, + "learning_rate": 1.8000000000000001e-06, + "loss": 0.0, + "reward": -0.09716667234897614, + "reward_std": 0.2380087822675705, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.09716667234897614, + "step": 9 + }, + { + "completion_length": 152.5, + "epoch": 0.0013381506757660913, + "grad_norm": 0.8989537358283997, + "kl": 0.0005690781399607658, + "learning_rate": 2.0000000000000003e-06, + "loss": 0.0, + "reward": 0.33650001883506775, + "reward_std": 1.0737173557281494, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08016666769981384, + "step": 10 + }, + { + "completion_length": 163.5, + "epoch": 0.0014719657433427003, + "grad_norm": 0.4434801936149597, + "kl": 0.0005024907295592129, + "learning_rate": 2.2e-06, + "loss": 0.0, + "reward": 0.3734999895095825, + "reward_std": 0.9148844480514526, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.043166667222976685, + "step": 11 + }, + { + "completion_length": 166.33334350585938, + "epoch": 0.0016057808109193096, + "grad_norm": 0.2695975601673126, + "kl": 0.0002385280531598255, + "learning_rate": 2.4000000000000003e-06, + "loss": 0.0, + "reward": -0.03350000083446503, + "reward_std": 0.0820579081773758, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03350000083446503, + "step": 12 + }, + { + "completion_length": 162.0, + "epoch": 0.0017395958784959186, + "grad_norm": 0.3843610882759094, + "kl": 0.0003402562579140067, + "learning_rate": 2.6e-06, + "loss": 0.0, + "reward": -0.1420000046491623, + "reward_std": 0.15615761280059814, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.1420000046491623, + "step": 13 + }, + { + "completion_length": 139.0, + "epoch": 0.0018734109460725277, + "grad_norm": 0.31161829829216003, + "kl": 0.00037740400875918567, + "learning_rate": 2.8000000000000003e-06, + "loss": 0.0, + "reward": -0.05516666918992996, + "reward_std": 0.10250934958457947, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.05516666918992996, + "step": 14 + }, + { + "completion_length": 198.5, + "epoch": 0.002007226013649137, + "grad_norm": 0.001213455805554986, + "kl": 0.0005896758520975709, + "learning_rate": 3e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 15 + }, + { + "completion_length": 142.5, + "epoch": 0.002141041081225746, + "grad_norm": 0.40445369482040405, + "kl": 0.00034244018024764955, + "learning_rate": 3.2000000000000003e-06, + "loss": 0.0, + "reward": 0.746666669845581, + "reward_std": 1.3729774951934814, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08666666597127914, + "step": 16 + }, + { + "completion_length": 193.1666717529297, + "epoch": 0.002274856148802355, + "grad_norm": 0.0018463014857843518, + "kl": 0.0003552571579348296, + "learning_rate": 3.4000000000000005e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 17 + }, + { + "completion_length": 165.6666717529297, + "epoch": 0.0024086712163789645, + "grad_norm": 0.390596479177475, + "kl": 0.00029498100047931075, + "learning_rate": 3.6000000000000003e-06, + "loss": 0.0, + "reward": -0.09133332967758179, + "reward_std": 0.14864006638526917, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.09133332967758179, + "step": 18 + }, + { + "completion_length": 175.33334350585938, + "epoch": 0.0025424862839555735, + "grad_norm": 0.29836106300354004, + "kl": 0.00035133626079186797, + "learning_rate": 3.8000000000000005e-06, + "loss": 0.0, + "reward": 0.044999998062849045, + "reward_std": 0.11022703349590302, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.038333334028720856, + "step": 19 + }, + { + "completion_length": 174.33334350585938, + "epoch": 0.0026763013515321826, + "grad_norm": 0.4651794135570526, + "kl": 0.000449147482868284, + "learning_rate": 4.000000000000001e-06, + "loss": 0.0, + "reward": 0.011333334259688854, + "reward_std": 0.060777194797992706, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.011333334259688854, + "step": 20 + }, + { + "completion_length": 188.5, + "epoch": 0.0028101164191087916, + "grad_norm": 0.45565637946128845, + "kl": 0.00044950511073693633, + "learning_rate": 4.2000000000000004e-06, + "loss": 0.0, + "reward": 0.04633333534002304, + "reward_std": 0.11349302530288696, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03700000047683716, + "step": 21 + }, + { + "completion_length": 168.33334350585938, + "epoch": 0.0029439314866854006, + "grad_norm": 0.36476781964302063, + "kl": 0.0003528912493493408, + "learning_rate": 4.4e-06, + "loss": 0.0, + "reward": 0.3645000159740448, + "reward_std": 0.8928390741348267, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.052166666835546494, + "step": 22 + }, + { + "completion_length": 118.83333587646484, + "epoch": 0.00307774655426201, + "grad_norm": 0.6629476547241211, + "kl": 0.00042950891656801105, + "learning_rate": 4.600000000000001e-06, + "loss": 0.0, + "reward": 0.013500000350177288, + "reward_std": 0.03306811302900314, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.013500000350177288, + "step": 23 + }, + { + "completion_length": 112.83333587646484, + "epoch": 0.003211561621838619, + "grad_norm": 0.49353620409965515, + "kl": 0.0005617816932499409, + "learning_rate": 4.800000000000001e-06, + "loss": 0.0, + "reward": 0.3318333327770233, + "reward_std": 1.0452561378479004, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.08483333885669708, + "step": 24 + }, + { + "completion_length": 131.1666717529297, + "epoch": 0.003345376689415228, + "grad_norm": 0.38862261176109314, + "kl": 0.0004901737556792796, + "learning_rate": 5e-06, + "loss": 0.0, + "reward": 0.3465000092983246, + "reward_std": 0.9767013788223267, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07016666978597641, + "step": 25 + }, + { + "completion_length": 128.5, + "epoch": 0.0034791917569918372, + "grad_norm": 0.41505903005599976, + "kl": 0.00022011167311575264, + "learning_rate": 4.999756310023261e-06, + "loss": 0.0, + "reward": 0.41233333945274353, + "reward_std": 1.010006308555603, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.00433333357796073, + "step": 26 + }, + { + "completion_length": 169.0, + "epoch": 0.0036130068245684463, + "grad_norm": 0.4222584664821625, + "kl": 0.0006107889348641038, + "learning_rate": 4.999025287600886e-06, + "loss": 0.0, + "reward": 0.021000001579523087, + "reward_std": 0.05143928527832031, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.021000001579523087, + "step": 27 + }, + { + "completion_length": 128.0, + "epoch": 0.0037468218921450553, + "grad_norm": 0.4782092273235321, + "kl": 0.000323088257573545, + "learning_rate": 4.997807075247147e-06, + "loss": 0.0, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 28 + }, + { + "completion_length": 200.0, + "epoch": 0.003880636959721665, + "grad_norm": 0.0008450484601780772, + "kl": 0.0003519197925925255, + "learning_rate": 4.996101910454953e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 29 + }, + { + "completion_length": 175.33334350585938, + "epoch": 0.004014452027298274, + "grad_norm": 0.33603161573410034, + "kl": 0.0004287260235287249, + "learning_rate": 4.993910125649561e-06, + "loss": 0.0, + "reward": -0.0560000017285347, + "reward_std": 0.1371714174747467, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.0560000017285347, + "step": 30 + }, + { + "completion_length": 168.83334350585938, + "epoch": 0.004148267094874883, + "grad_norm": 0.29343071579933167, + "kl": 0.0004830901452805847, + "learning_rate": 4.9912321481237616e-06, + "loss": 0.0, + "reward": 0.04183333367109299, + "reward_std": 0.064808689057827, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.04183333367109299, + "step": 31 + }, + { + "completion_length": 149.0, + "epoch": 0.004282082162451492, + "grad_norm": 0.4075845181941986, + "kl": 0.0004737289564218372, + "learning_rate": 4.988068499954578e-06, + "loss": 0.0, + "reward": 0.36633333563804626, + "reward_std": 0.8973297476768494, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.050333332270383835, + "step": 32 + }, + { + "completion_length": 145.6666717529297, + "epoch": 0.004415897230028101, + "grad_norm": 0.5869320631027222, + "kl": 0.00045360170770436525, + "learning_rate": 4.984419797901491e-06, + "loss": 0.0, + "reward": -0.058666668832302094, + "reward_std": 0.12906846404075623, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.058666668832302094, + "step": 33 + }, + { + "completion_length": 188.0, + "epoch": 0.00454971229760471, + "grad_norm": 0.44135627150535583, + "kl": 0.0004000771732535213, + "learning_rate": 4.980286753286196e-06, + "loss": 0.0, + "reward": 0.040666669607162476, + "reward_std": 0.06300053000450134, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.040666669607162476, + "step": 34 + }, + { + "completion_length": 186.1666717529297, + "epoch": 0.004683527365181319, + "grad_norm": 0.29517051577568054, + "kl": 0.0003852550871670246, + "learning_rate": 4.975670171853926e-06, + "loss": 0.0, + "reward": 0.0416666679084301, + "reward_std": 0.10206206887960434, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0416666679084301, + "step": 35 + }, + { + "completion_length": 198.1666717529297, + "epoch": 0.004817342432757929, + "grad_norm": 0.004241094458848238, + "kl": 0.000525132636539638, + "learning_rate": 4.970570953616383e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 36 + }, + { + "completion_length": 111.16667175292969, + "epoch": 0.004951157500334538, + "grad_norm": 0.9546847343444824, + "kl": 0.0008722383063286543, + "learning_rate": 4.964990092676263e-06, + "loss": 0.0, + "reward": 0.00433333357796073, + "reward_std": 0.07178208231925964, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.00433333357796073, + "step": 37 + }, + { + "completion_length": 116.33333587646484, + "epoch": 0.005084972567911147, + "grad_norm": 0.4917941093444824, + "kl": 0.0006737456424161792, + "learning_rate": 4.958928677033465e-06, + "loss": 0.0, + "reward": 0.4699999988079071, + "reward_std": 0.9978088140487671, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0533333346247673, + "step": 38 + }, + { + "completion_length": 136.33334350585938, + "epoch": 0.005218787635487756, + "grad_norm": 0.4850587248802185, + "kl": 0.0006846442702226341, + "learning_rate": 4.9523878883729794e-06, + "loss": 0.0, + "reward": 1.321166753768921, + "reward_std": 1.2952899932861328, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.01216666679829359, + "step": 39 + }, + { + "completion_length": 170.5, + "epoch": 0.005352602703064365, + "grad_norm": 0.0013024378567934036, + "kl": 0.00040273740887641907, + "learning_rate": 4.9453690018345144e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 40 + }, + { + "completion_length": 173.6666717529297, + "epoch": 0.005486417770640974, + "grad_norm": 0.5376860499382019, + "kl": 0.0011046230792999268, + "learning_rate": 4.937873385763909e-06, + "loss": 0.0, + "reward": 0.7595000267028809, + "reward_std": 1.2757749557495117, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07383333891630173, + "step": 41 + }, + { + "completion_length": 172.33334350585938, + "epoch": 0.005620232838217583, + "grad_norm": 0.48040255904197693, + "kl": 0.001196104334667325, + "learning_rate": 4.9299025014463665e-06, + "loss": 0.0, + "reward": 0.7818333506584167, + "reward_std": 1.171539306640625, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.05150000378489494, + "step": 42 + }, + { + "completion_length": 200.0, + "epoch": 0.005754047905794192, + "grad_norm": 0.0011990669881924987, + "kl": 0.00046603806549683213, + "learning_rate": 4.921457902821578e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 43 + }, + { + "completion_length": 167.1666717529297, + "epoch": 0.005887862973370801, + "grad_norm": 0.002651863731443882, + "kl": 0.0007552001625299454, + "learning_rate": 4.912541236180779e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 44 + }, + { + "completion_length": 180.0, + "epoch": 0.00602167804094741, + "grad_norm": 0.3888886868953705, + "kl": 0.000817882944829762, + "learning_rate": 4.903154239845798e-06, + "loss": 0.0, + "reward": 1.2708333730697632, + "reward_std": 1.3472579717636108, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 45 + }, + { + "completion_length": 108.66667175292969, + "epoch": 0.00615549310852402, + "grad_norm": 0.0016210161847993731, + "kl": 0.00039254926377907395, + "learning_rate": 4.893298743830168e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 46 + }, + { + "completion_length": 154.6666717529297, + "epoch": 0.006289308176100629, + "grad_norm": 0.341744601726532, + "kl": 0.0009102068725042045, + "learning_rate": 4.882976669482368e-06, + "loss": 0.0, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 47 + }, + { + "completion_length": 126.16667175292969, + "epoch": 0.006423123243677238, + "grad_norm": 0.012836282141506672, + "kl": 0.0018564595375210047, + "learning_rate": 4.8721900291112415e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 48 + }, + { + "completion_length": 101.83333587646484, + "epoch": 0.006556938311253847, + "grad_norm": 0.6613649725914001, + "kl": 0.0017794461455196142, + "learning_rate": 4.860940925593703e-06, + "loss": 0.0001, + "reward": 0.04200000315904617, + "reward_std": 0.06506612151861191, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.04200000315904617, + "step": 49 + }, + { + "completion_length": 196.5, + "epoch": 0.006690753378830456, + "grad_norm": 0.004236978013068438, + "kl": 0.0008604646427556872, + "learning_rate": 4.849231551964771e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 50 + }, + { + "completion_length": 87.33333587646484, + "epoch": 0.006824568446407065, + "grad_norm": 2.5036542415618896, + "kl": 0.002497585490345955, + "learning_rate": 4.837064190990036e-06, + "loss": 0.0001, + "reward": 0.020333334803581238, + "reward_std": 0.04980628937482834, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.020333334803581238, + "step": 51 + }, + { + "completion_length": 130.6666717529297, + "epoch": 0.0069583835139836745, + "grad_norm": 0.4602515697479248, + "kl": 0.0022823391482234, + "learning_rate": 4.824441214720629e-06, + "loss": 0.0001, + "reward": 0.8040000200271606, + "reward_std": 1.2467942237854004, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.029333334416151047, + "step": 52 + }, + { + "completion_length": 113.0, + "epoch": 0.0070921985815602835, + "grad_norm": 0.5198715329170227, + "kl": 0.005426853429526091, + "learning_rate": 4.811365084030784e-06, + "loss": 0.0002, + "reward": 1.254666805267334, + "reward_std": 1.3316380977630615, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.004666668828576803, + "step": 53 + }, + { + "completion_length": 154.6666717529297, + "epoch": 0.0072260136491368926, + "grad_norm": 0.34585776925086975, + "kl": 0.002139053540304303, + "learning_rate": 4.7978383481380865e-06, + "loss": 0.0001, + "reward": 1.2051666975021362, + "reward_std": 1.323843002319336, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.04483333230018616, + "step": 54 + }, + { + "completion_length": 187.33334350585938, + "epoch": 0.007359828716713502, + "grad_norm": 0.4027135670185089, + "kl": 0.0035339975729584694, + "learning_rate": 4.783863644106502e-06, + "loss": 0.0001, + "reward": 0.9166666865348816, + "reward_std": 1.2416388988494873, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 55 + }, + { + "completion_length": 195.6666717529297, + "epoch": 0.007493643784290111, + "grad_norm": 0.00830562599003315, + "kl": 0.00114604108966887, + "learning_rate": 4.769443696332272e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 56 + }, + { + "completion_length": 129.5, + "epoch": 0.0076274588518667205, + "grad_norm": 0.5255561470985413, + "kl": 0.0031108385883271694, + "learning_rate": 4.754581316012785e-06, + "loss": 0.0001, + "reward": 0.4346666932106018, + "reward_std": 1.0144504308700562, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.017999999225139618, + "step": 57 + }, + { + "completion_length": 159.83334350585938, + "epoch": 0.00776127391944333, + "grad_norm": 0.46986404061317444, + "kl": 0.00284760445356369, + "learning_rate": 4.7392794005985324e-06, + "loss": 0.0001, + "reward": 0.04800000041723251, + "reward_std": 0.07532595098018646, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.035333335399627686, + "step": 58 + }, + { + "completion_length": 121.66667175292969, + "epoch": 0.007895088987019938, + "grad_norm": 1.0778398513793945, + "kl": 0.003858643351122737, + "learning_rate": 4.723540933228245e-06, + "loss": 0.0002, + "reward": 0.0338333360850811, + "reward_std": 0.053443118929862976, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0338333360850811, + "step": 59 + }, + { + "completion_length": 169.83334350585938, + "epoch": 0.008028904054596548, + "grad_norm": 0.3702806830406189, + "kl": 0.003246339038014412, + "learning_rate": 4.707368982147318e-06, + "loss": 0.0001, + "reward": -0.0560000017285347, + "reward_std": 0.1371714174747467, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.0560000017285347, + "step": 60 + }, + { + "completion_length": 162.0, + "epoch": 0.008162719122173156, + "grad_norm": 0.004582512192428112, + "kl": 0.0006104764179326594, + "learning_rate": 4.690766700109659e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 61 + }, + { + "completion_length": 175.1666717529297, + "epoch": 0.008296534189749766, + "grad_norm": 0.4721982181072235, + "kl": 0.0031741862185299397, + "learning_rate": 4.673737323763048e-06, + "loss": 0.0001, + "reward": 0.8009999990463257, + "reward_std": 1.2424194812774658, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.032333336770534515, + "step": 62 + }, + { + "completion_length": 122.5, + "epoch": 0.008430349257326376, + "grad_norm": 0.022069407626986504, + "kl": 0.007255158387124538, + "learning_rate": 4.656284173018144e-06, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 63 + }, + { + "completion_length": 150.5, + "epoch": 0.008564164324902984, + "grad_norm": 0.0063232239335775375, + "kl": 0.005966612603515387, + "learning_rate": 4.638410650401267e-06, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 64 + }, + { + "completion_length": 200.0, + "epoch": 0.008697979392479594, + "grad_norm": 0.0014898076187819242, + "kl": 0.0004787092038895935, + "learning_rate": 4.620120240391065e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 65 + }, + { + "completion_length": 172.5, + "epoch": 0.008831794460056202, + "grad_norm": 0.2924825847148895, + "kl": 0.0044395048171281815, + "learning_rate": 4.601416508739211e-06, + "loss": 0.0002, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 66 + }, + { + "completion_length": 162.0, + "epoch": 0.008965609527632812, + "grad_norm": 0.008148525841534138, + "kl": 0.0034142467193305492, + "learning_rate": 4.582303101775249e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 67 + }, + { + "completion_length": 165.83334350585938, + "epoch": 0.00909942459520942, + "grad_norm": 0.38308584690093994, + "kl": 0.00867555383592844, + "learning_rate": 4.562783745695738e-06, + "loss": 0.0003, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 68 + }, + { + "completion_length": 134.6666717529297, + "epoch": 0.00923323966278603, + "grad_norm": 0.450452595949173, + "kl": 0.005564470775425434, + "learning_rate": 4.542862245837821e-06, + "loss": 0.0002, + "reward": -0.03816666826605797, + "reward_std": 0.09348885715007782, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03816666826605797, + "step": 69 + }, + { + "completion_length": 166.33334350585938, + "epoch": 0.009367054730362638, + "grad_norm": 0.4226210415363312, + "kl": 0.0031330427154898643, + "learning_rate": 4.522542485937369e-06, + "loss": 0.0001, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 70 + }, + { + "completion_length": 196.33334350585938, + "epoch": 0.009500869797939248, + "grad_norm": 0.2609606683254242, + "kl": 0.00573092233389616, + "learning_rate": 4.501828427371834e-06, + "loss": 0.0002, + "reward": 0.4375, + "reward_std": 1.0116509199142456, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 71 + }, + { + "completion_length": 183.33334350585938, + "epoch": 0.009634684865515858, + "grad_norm": 0.001673938357271254, + "kl": 0.0006895886617712677, + "learning_rate": 4.4807241083879774e-06, + "loss": 0.0, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 72 + }, + { + "completion_length": 183.33334350585938, + "epoch": 0.009768499933092466, + "grad_norm": 0.41474249958992004, + "kl": 0.003182145766913891, + "learning_rate": 4.4592336433146e-06, + "loss": 0.0001, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 73 + }, + { + "completion_length": 198.5, + "epoch": 0.009902315000669076, + "grad_norm": 0.016794368624687195, + "kl": 0.0026337592862546444, + "learning_rate": 4.437361221760449e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 74 + }, + { + "completion_length": 180.5, + "epoch": 0.010036130068245684, + "grad_norm": 0.2793622314929962, + "kl": 0.008324583992362022, + "learning_rate": 4.415111107797445e-06, + "loss": 0.0003, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 75 + }, + { + "completion_length": 106.83333587646484, + "epoch": 0.010169945135822294, + "grad_norm": 0.6471434831619263, + "kl": 0.016430877149105072, + "learning_rate": 4.3924876391293915e-06, + "loss": 0.0007, + "reward": 0.021000001579523087, + "reward_std": 0.05143928527832031, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.021000001579523087, + "step": 76 + }, + { + "completion_length": 127.33333587646484, + "epoch": 0.010303760203398902, + "grad_norm": 0.3994431793689728, + "kl": 0.006109229288995266, + "learning_rate": 4.36949522624633e-06, + "loss": 0.0002, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 77 + }, + { + "completion_length": 133.1666717529297, + "epoch": 0.010437575270975512, + "grad_norm": 0.5078148245811462, + "kl": 0.012061258777976036, + "learning_rate": 4.346138351564711e-06, + "loss": 0.0005, + "reward": 1.628499984741211, + "reward_std": 1.352062702178955, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03816666826605797, + "step": 78 + }, + { + "completion_length": 102.83333587646484, + "epoch": 0.01057139033855212, + "grad_norm": 0.029442179948091507, + "kl": 0.010023966431617737, + "learning_rate": 4.322421568553529e-06, + "loss": 0.0004, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 79 + }, + { + "completion_length": 171.5, + "epoch": 0.01070520540612873, + "grad_norm": 0.608511745929718, + "kl": 0.02281329035758972, + "learning_rate": 4.2983495008466285e-06, + "loss": 0.0009, + "reward": -0.07016666978597641, + "reward_std": 0.10962922871112823, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07016666978597641, + "step": 80 + }, + { + "completion_length": 191.5, + "epoch": 0.010839020473705338, + "grad_norm": 0.30423974990844727, + "kl": 0.008041778579354286, + "learning_rate": 4.273926841341303e-06, + "loss": 0.0003, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 81 + }, + { + "completion_length": 114.83333587646484, + "epoch": 0.010972835541281948, + "grad_norm": 0.010348974727094173, + "kl": 0.01861591637134552, + "learning_rate": 4.249158351283414e-06, + "loss": 0.0007, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 82 + }, + { + "completion_length": 163.5, + "epoch": 0.011106650608858558, + "grad_norm": 0.38949424028396606, + "kl": 0.014888926409184933, + "learning_rate": 4.224048859339175e-06, + "loss": 0.0006, + "reward": 1.185666799545288, + "reward_std": 1.302016019821167, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.06433333456516266, + "step": 83 + }, + { + "completion_length": 180.5, + "epoch": 0.011240465676435166, + "grad_norm": 4.860539436340332, + "kl": 0.018028978258371353, + "learning_rate": 4.198603260653792e-06, + "loss": 0.0007, + "reward": 0.875, + "reward_std": 1.259960412979126, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0416666679084301, + "step": 84 + }, + { + "completion_length": 160.6666717529297, + "epoch": 0.011374280744011776, + "grad_norm": 0.45955681800842285, + "kl": 0.014035634696483612, + "learning_rate": 4.172826515897146e-06, + "loss": 0.0006, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 85 + }, + { + "completion_length": 146.33334350585938, + "epoch": 0.011508095811588384, + "grad_norm": 0.40156418085098267, + "kl": 0.011429629288613796, + "learning_rate": 4.146723650296701e-06, + "loss": 0.0005, + "reward": -0.03099999576807022, + "reward_std": 0.208724707365036, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03099999949336052, + "step": 86 + }, + { + "completion_length": 120.0, + "epoch": 0.011641910879164994, + "grad_norm": 0.5004826784133911, + "kl": 0.021759724244475365, + "learning_rate": 4.120299752657828e-06, + "loss": 0.0009, + "reward": 1.6751668453216553, + "reward_std": 1.2977288961410522, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.008500000461935997, + "step": 87 + }, + { + "completion_length": 188.0, + "epoch": 0.011775725946741603, + "grad_norm": 0.446821391582489, + "kl": 0.009519928134977818, + "learning_rate": 4.093559974371725e-06, + "loss": 0.0004, + "reward": 0.02083333395421505, + "reward_std": 0.05103103443980217, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 88 + }, + { + "completion_length": 171.6666717529297, + "epoch": 0.011909541014318212, + "grad_norm": 0.003901235293596983, + "kl": 0.010121981613337994, + "learning_rate": 4.066509528411151e-06, + "loss": 0.0004, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 89 + }, + { + "completion_length": 154.5, + "epoch": 0.01204335608189482, + "grad_norm": 0.5476510524749756, + "kl": 0.022881466895341873, + "learning_rate": 4.039153688314146e-06, + "loss": 0.0009, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 90 + }, + { + "completion_length": 176.6666717529297, + "epoch": 0.01217717114947143, + "grad_norm": 0.8005715608596802, + "kl": 0.02562887594103813, + "learning_rate": 4.011497787155938e-06, + "loss": 0.001, + "reward": 0.02083333395421505, + "reward_std": 0.05103103816509247, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 91 + }, + { + "completion_length": 157.83334350585938, + "epoch": 0.01231098621704804, + "grad_norm": 0.572854220867157, + "kl": 0.010252588428556919, + "learning_rate": 3.983547216509254e-06, + "loss": 0.0004, + "reward": 0.0833333358168602, + "reward_std": 0.20412415266036987, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 92 + }, + { + "completion_length": 168.0, + "epoch": 0.012444801284624649, + "grad_norm": 0.3673333525657654, + "kl": 0.008399921469390392, + "learning_rate": 3.955307425393224e-06, + "loss": 0.0003, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 93 + }, + { + "completion_length": 167.0, + "epoch": 0.012578616352201259, + "grad_norm": 0.028444070369005203, + "kl": 0.011543246917426586, + "learning_rate": 3.92678391921108e-06, + "loss": 0.0005, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 94 + }, + { + "completion_length": 98.33333587646484, + "epoch": 0.012712431419777867, + "grad_norm": 0.452796995639801, + "kl": 0.020905088633298874, + "learning_rate": 3.897982258676867e-06, + "loss": 0.0008, + "reward": 0.75, + "reward_std": 0.8803408145904541, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 95 + }, + { + "completion_length": 166.6666717529297, + "epoch": 0.012846246487354477, + "grad_norm": 0.322458416223526, + "kl": 0.013159887865185738, + "learning_rate": 3.868908058731376e-06, + "loss": 0.0005, + "reward": 2.06000018119812, + "reward_std": 1.010742425918579, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.023333333432674408, + "step": 96 + }, + { + "completion_length": 177.33334350585938, + "epoch": 0.012980061554931085, + "grad_norm": 0.3679336905479431, + "kl": 0.009648090228438377, + "learning_rate": 3.839566987447492e-06, + "loss": 0.0004, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 97 + }, + { + "completion_length": 188.6666717529297, + "epoch": 0.013113876622507695, + "grad_norm": 0.004214553628116846, + "kl": 0.0036848734598606825, + "learning_rate": 3.8099647649251984e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 98 + }, + { + "completion_length": 145.83334350585938, + "epoch": 0.013247691690084303, + "grad_norm": 0.5636357069015503, + "kl": 0.01897261291742325, + "learning_rate": 3.780107162176429e-06, + "loss": 0.0008, + "reward": 0.9166666865348816, + "reward_std": 1.2416388988494873, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 99 + }, + { + "completion_length": 126.0, + "epoch": 0.013381506757660913, + "grad_norm": 0.5204256772994995, + "kl": 0.026616668328642845, + "learning_rate": 3.7500000000000005e-06, + "loss": 0.0011, + "reward": 1.2710000276565552, + "reward_std": 1.347088098526001, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.021000001579523087, + "step": 100 + }, + { + "completion_length": 199.1666717529297, + "epoch": 0.013515321825237521, + "grad_norm": 0.4379250407218933, + "kl": 0.004337076563388109, + "learning_rate": 3.7196491478468322e-06, + "loss": 0.0002, + "reward": 0.02083333395421505, + "reward_std": 0.05103103443980217, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 101 + }, + { + "completion_length": 200.0, + "epoch": 0.01364913689281413, + "grad_norm": 0.464220255613327, + "kl": 0.002902195556089282, + "learning_rate": 3.689060522675689e-06, + "loss": 0.0001, + "reward": 0.02083333395421505, + "reward_std": 0.05103103443980217, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 102 + }, + { + "completion_length": 162.5, + "epoch": 0.01378295196039074, + "grad_norm": 0.4563305079936981, + "kl": 0.031100619584321976, + "learning_rate": 3.658240087799655e-06, + "loss": 0.0012, + "reward": 1.6421668529510498, + "reward_std": 1.2732903957366943, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.024500001221895218, + "step": 103 + }, + { + "completion_length": 148.6666717529297, + "epoch": 0.013916767027967349, + "grad_norm": 0.03375934436917305, + "kl": 0.02203848585486412, + "learning_rate": 3.627193851723577e-06, + "loss": 0.0009, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 104 + }, + { + "completion_length": 181.6666717529297, + "epoch": 0.014050582095543959, + "grad_norm": 0.34021925926208496, + "kl": 0.013872651383280754, + "learning_rate": 3.595927866972694e-06, + "loss": 0.0006, + "reward": 0.9375, + "reward_std": 1.224106788635254, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 105 + }, + { + "completion_length": 148.33334350585938, + "epoch": 0.014184397163120567, + "grad_norm": 0.4161204695701599, + "kl": 0.011628372594714165, + "learning_rate": 3.564448228912682e-06, + "loss": 0.0005, + "reward": 1.75, + "reward_std": 1.172603964805603, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 106 + }, + { + "completion_length": 199.83334350585938, + "epoch": 0.014318212230697177, + "grad_norm": 0.2740929126739502, + "kl": 0.0027336678467690945, + "learning_rate": 3.532761074561355e-06, + "loss": 0.0001, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 107 + }, + { + "completion_length": 199.6666717529297, + "epoch": 0.014452027298273785, + "grad_norm": 0.34659114480018616, + "kl": 0.003705874551087618, + "learning_rate": 3.5008725813922383e-06, + "loss": 0.0001, + "reward": 0.0833333358168602, + "reward_std": 0.20412413775920868, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 108 + }, + { + "completion_length": 181.83334350585938, + "epoch": 0.014585842365850395, + "grad_norm": 0.03158266097307205, + "kl": 0.016718031838536263, + "learning_rate": 3.4687889661302577e-06, + "loss": 0.0007, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 109 + }, + { + "completion_length": 125.0, + "epoch": 0.014719657433427003, + "grad_norm": 0.48598426580429077, + "kl": 0.05068827420473099, + "learning_rate": 3.436516483539781e-06, + "loss": 0.002, + "reward": 1.7691667079925537, + "reward_std": 1.1387290954589844, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.019166667014360428, + "step": 110 + }, + { + "completion_length": 181.6666717529297, + "epoch": 0.014853472501003613, + "grad_norm": 0.33522072434425354, + "kl": 0.007758219726383686, + "learning_rate": 3.4040614252052305e-06, + "loss": 0.0003, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 111 + }, + { + "completion_length": 185.83334350585938, + "epoch": 0.014987287568580221, + "grad_norm": 0.35056257247924805, + "kl": 0.015054799616336823, + "learning_rate": 3.3714301183045382e-06, + "loss": 0.0006, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 112 + }, + { + "completion_length": 183.33334350585938, + "epoch": 0.015121102636156831, + "grad_norm": 0.32826197147369385, + "kl": 0.04551512748003006, + "learning_rate": 3.338628924375638e-06, + "loss": 0.0018, + "reward": 0.41050001978874207, + "reward_std": 1.034417986869812, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.006166668143123388, + "step": 113 + }, + { + "completion_length": 174.83334350585938, + "epoch": 0.015254917703733441, + "grad_norm": 0.3377326726913452, + "kl": 0.04113561660051346, + "learning_rate": 3.3056642380762783e-06, + "loss": 0.0016, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 114 + }, + { + "completion_length": 150.6666717529297, + "epoch": 0.01538873277131005, + "grad_norm": 0.6040213108062744, + "kl": 0.021824371069669724, + "learning_rate": 3.272542485937369e-06, + "loss": 0.0009, + "reward": 0.8681666851043701, + "reward_std": 1.2657527923583984, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.034833334386348724, + "step": 115 + }, + { + "completion_length": 200.0, + "epoch": 0.01552254783888666, + "grad_norm": 0.008158405311405659, + "kl": 0.0036414554342627525, + "learning_rate": 3.2392701251101172e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 116 + }, + { + "completion_length": 200.0, + "epoch": 0.01565636290646327, + "grad_norm": 0.003316780785098672, + "kl": 0.0018421653658151627, + "learning_rate": 3.205853642107192e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 117 + }, + { + "completion_length": 156.6666717529297, + "epoch": 0.015790177974039876, + "grad_norm": 0.5106171369552612, + "kl": 0.028099259361624718, + "learning_rate": 3.1722995515381644e-06, + "loss": 0.0011, + "reward": 1.2643333673477173, + "reward_std": 1.354689359664917, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.014333332888782024, + "step": 118 + }, + { + "completion_length": 139.33334350585938, + "epoch": 0.015923993041616485, + "grad_norm": 0.3868845999240875, + "kl": 0.05956920236349106, + "learning_rate": 3.1386143948394764e-06, + "loss": 0.0024, + "reward": 1.687666654586792, + "reward_std": 1.259092092514038, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.021000001579523087, + "step": 119 + }, + { + "completion_length": 197.0, + "epoch": 0.016057808109193095, + "grad_norm": 0.3134411573410034, + "kl": 0.021013274788856506, + "learning_rate": 3.1048047389991693e-06, + "loss": 0.0008, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 120 + }, + { + "completion_length": 179.1666717529297, + "epoch": 0.016191623176769705, + "grad_norm": 0.636957585811615, + "kl": 0.024236779659986496, + "learning_rate": 3.0708771752766397e-06, + "loss": 0.001, + "reward": 0.0833333358168602, + "reward_std": 0.20412413775920868, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 121 + }, + { + "completion_length": 115.83333587646484, + "epoch": 0.01632543824434631, + "grad_norm": 0.01371024176478386, + "kl": 0.042789995670318604, + "learning_rate": 3.0368383179176584e-06, + "loss": 0.0017, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 122 + }, + { + "completion_length": 122.5, + "epoch": 0.01645925331192292, + "grad_norm": 0.005612315144389868, + "kl": 0.020341983065009117, + "learning_rate": 3.002694802864912e-06, + "loss": 0.0008, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 123 + }, + { + "completion_length": 194.6666717529297, + "epoch": 0.01659306837949953, + "grad_norm": 0.00693311495706439, + "kl": 0.007592849433422089, + "learning_rate": 2.9684532864643123e-06, + "loss": 0.0003, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 124 + }, + { + "completion_length": 106.5, + "epoch": 0.01672688344707614, + "grad_norm": 0.013540594838559628, + "kl": 0.027746539562940598, + "learning_rate": 2.9341204441673267e-06, + "loss": 0.0011, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 125 + }, + { + "completion_length": 194.1666717529297, + "epoch": 0.01686069851465275, + "grad_norm": 0.38014551997184753, + "kl": 0.017290499061346054, + "learning_rate": 2.8997029692295875e-06, + "loss": 0.0007, + "reward": 0.9166666865348816, + "reward_std": 1.2416388988494873, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 126 + }, + { + "completion_length": 175.5, + "epoch": 0.016994513582229358, + "grad_norm": 0.4393642842769623, + "kl": 0.009731654077768326, + "learning_rate": 2.8652075714060296e-06, + "loss": 0.0004, + "reward": 0.9166666865348816, + "reward_std": 1.2416388988494873, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 127 + }, + { + "completion_length": 108.5, + "epoch": 0.017128328649805968, + "grad_norm": 0.016502853482961655, + "kl": 0.045004379004240036, + "learning_rate": 2.8306409756428067e-06, + "loss": 0.0018, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 128 + }, + { + "completion_length": 126.16667175292969, + "epoch": 0.017262143717382578, + "grad_norm": 0.34395888447761536, + "kl": 0.016212839633226395, + "learning_rate": 2.7960099207662535e-06, + "loss": 0.0006, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 129 + }, + { + "completion_length": 154.6666717529297, + "epoch": 0.017395958784959187, + "grad_norm": 0.4351252615451813, + "kl": 0.02109598182141781, + "learning_rate": 2.761321158169134e-06, + "loss": 0.0008, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 130 + }, + { + "completion_length": 91.16667175292969, + "epoch": 0.017529773852535794, + "grad_norm": 0.5746130347251892, + "kl": 0.0653064101934433, + "learning_rate": 2.726581450494451e-06, + "loss": 0.0026, + "reward": 2.454500198364258, + "reward_std": 0.11145174503326416, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.045500002801418304, + "step": 131 + }, + { + "completion_length": 121.66667175292969, + "epoch": 0.017663588920112404, + "grad_norm": 0.007989317178726196, + "kl": 0.03470905125141144, + "learning_rate": 2.6917975703170466e-06, + "loss": 0.0014, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 132 + }, + { + "completion_length": 158.33334350585938, + "epoch": 0.017797403987689014, + "grad_norm": 0.3317304849624634, + "kl": 0.014697610400617123, + "learning_rate": 2.6569762988232838e-06, + "loss": 0.0006, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 133 + }, + { + "completion_length": 185.5, + "epoch": 0.017931219055265624, + "grad_norm": 0.4038233458995819, + "kl": 0.007475182414054871, + "learning_rate": 2.6221244244890336e-06, + "loss": 0.0003, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 134 + }, + { + "completion_length": 147.33334350585938, + "epoch": 0.018065034122842234, + "grad_norm": 0.4337051510810852, + "kl": 0.03628496080636978, + "learning_rate": 2.587248741756253e-06, + "loss": 0.0015, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 135 + }, + { + "completion_length": 186.33334350585938, + "epoch": 0.01819884919041884, + "grad_norm": 0.430644690990448, + "kl": 0.029759211465716362, + "learning_rate": 2.5523560497083927e-06, + "loss": 0.0012, + "reward": 0.5, + "reward_std": 1.0, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 136 + }, + { + "completion_length": 121.66667175292969, + "epoch": 0.01833266425799545, + "grad_norm": 0.5082526803016663, + "kl": 0.050083570182323456, + "learning_rate": 2.517453150744904e-06, + "loss": 0.002, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 137 + }, + { + "completion_length": 189.33334350585938, + "epoch": 0.01846647932557206, + "grad_norm": 0.2880929112434387, + "kl": 0.010486051440238953, + "learning_rate": 2.482546849255096e-06, + "loss": 0.0004, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 138 + }, + { + "completion_length": 161.83334350585938, + "epoch": 0.01860029439314867, + "grad_norm": 0.012269456870853901, + "kl": 0.016117241233587265, + "learning_rate": 2.447643950291608e-06, + "loss": 0.0006, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 139 + }, + { + "completion_length": 146.6666717529297, + "epoch": 0.018734109460725276, + "grad_norm": 0.46656334400177, + "kl": 0.04306882619857788, + "learning_rate": 2.4127512582437486e-06, + "loss": 0.0017, + "reward": 0.7596666812896729, + "reward_std": 1.358882188796997, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.07366666942834854, + "step": 140 + }, + { + "completion_length": 183.1666717529297, + "epoch": 0.018867924528301886, + "grad_norm": 0.31509730219841003, + "kl": 0.018859118223190308, + "learning_rate": 2.377875575510967e-06, + "loss": 0.0008, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 141 + }, + { + "completion_length": 122.0, + "epoch": 0.019001739595878496, + "grad_norm": 0.01961562968790531, + "kl": 0.04471735656261444, + "learning_rate": 2.3430237011767166e-06, + "loss": 0.0018, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 142 + }, + { + "completion_length": 135.5, + "epoch": 0.019135554663455106, + "grad_norm": 0.41898852586746216, + "kl": 0.04450133442878723, + "learning_rate": 2.3082024296829538e-06, + "loss": 0.0018, + "reward": 0.4583333432674408, + "reward_std": 1.005194902420044, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0416666679084301, + "step": 143 + }, + { + "completion_length": 190.1666717529297, + "epoch": 0.019269369731031716, + "grad_norm": 0.6943901777267456, + "kl": 0.031737446784973145, + "learning_rate": 2.2734185495055503e-06, + "loss": 0.0013, + "reward": 0.02083333395421505, + "reward_std": 0.05103103443980217, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 144 + }, + { + "completion_length": 177.83334350585938, + "epoch": 0.019403184798608322, + "grad_norm": 0.4527323544025421, + "kl": 0.011997250840067863, + "learning_rate": 2.238678841830867e-06, + "loss": 0.0005, + "reward": 0.25, + "reward_std": 0.273861289024353, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 145 + }, + { + "completion_length": 94.33333587646484, + "epoch": 0.019536999866184932, + "grad_norm": 0.9129260182380676, + "kl": 0.03500860929489136, + "learning_rate": 2.2039900792337477e-06, + "loss": 0.0014, + "reward": 2.5158333778381348, + "reward_std": 0.038783639669418335, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.015833333134651184, + "step": 146 + }, + { + "completion_length": 151.0, + "epoch": 0.019670814933761542, + "grad_norm": 0.4710156321525574, + "kl": 0.040015410631895065, + "learning_rate": 2.1693590243571937e-06, + "loss": 0.0016, + "reward": 1.75, + "reward_std": 1.172603964805603, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 147 + }, + { + "completion_length": 109.5, + "epoch": 0.019804630001338152, + "grad_norm": 0.4614563584327698, + "kl": 0.046104662120342255, + "learning_rate": 2.134792428593971e-06, + "loss": 0.0018, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 148 + }, + { + "completion_length": 170.83334350585938, + "epoch": 0.01993844506891476, + "grad_norm": 0.37295976281166077, + "kl": 0.026096370071172714, + "learning_rate": 2.1002970307704134e-06, + "loss": 0.001, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 149 + }, + { + "completion_length": 141.5, + "epoch": 0.02007226013649137, + "grad_norm": 0.46510520577430725, + "kl": 0.04087626934051514, + "learning_rate": 2.0658795558326745e-06, + "loss": 0.0016, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 150 + }, + { + "completion_length": 196.0, + "epoch": 0.020206075204067978, + "grad_norm": 0.12236814945936203, + "kl": 0.03315367549657822, + "learning_rate": 2.031546713535688e-06, + "loss": 0.0013, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 151 + }, + { + "completion_length": 124.33333587646484, + "epoch": 0.020339890271644588, + "grad_norm": 0.4959777891635895, + "kl": 0.06505782902240753, + "learning_rate": 1.997305197135089e-06, + "loss": 0.0026, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 152 + }, + { + "completion_length": 98.33333587646484, + "epoch": 0.020473705339221198, + "grad_norm": 0.5583345293998718, + "kl": 0.06674438714981079, + "learning_rate": 1.963161682082342e-06, + "loss": 0.0027, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 153 + }, + { + "completion_length": 185.33334350585938, + "epoch": 0.020607520406797804, + "grad_norm": 0.4062897264957428, + "kl": 0.008387739770114422, + "learning_rate": 1.9291228247233607e-06, + "loss": 0.0003, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 154 + }, + { + "completion_length": 162.5, + "epoch": 0.020741335474374414, + "grad_norm": 0.483214795589447, + "kl": 0.04108166694641113, + "learning_rate": 1.895195261000831e-06, + "loss": 0.0016, + "reward": 0.9166666865348816, + "reward_std": 1.2416388988494873, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 155 + }, + { + "completion_length": 111.33333587646484, + "epoch": 0.020875150541951024, + "grad_norm": 1.3660439252853394, + "kl": 0.14076313376426697, + "learning_rate": 1.8613856051605242e-06, + "loss": 0.0056, + "reward": 2.5288333892822266, + "reward_std": 0.07062703371047974, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.028833333402872086, + "step": 156 + }, + { + "completion_length": 118.16667175292969, + "epoch": 0.021008965609527634, + "grad_norm": 0.5607126355171204, + "kl": 0.04791057109832764, + "learning_rate": 1.827700448461836e-06, + "loss": 0.0019, + "reward": 0.905666708946228, + "reward_std": 1.251636028289795, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.010999999940395355, + "step": 157 + }, + { + "completion_length": 93.5, + "epoch": 0.02114278067710424, + "grad_norm": 0.013210687786340714, + "kl": 0.02975599095225334, + "learning_rate": 1.7941463578928088e-06, + "loss": 0.0012, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 158 + }, + { + "completion_length": 184.83334350585938, + "epoch": 0.02127659574468085, + "grad_norm": 0.5084704160690308, + "kl": 0.01870289444923401, + "learning_rate": 1.7607298748898844e-06, + "loss": 0.0007, + "reward": 0.25, + "reward_std": 0.273861289024353, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 159 + }, + { + "completion_length": 148.1666717529297, + "epoch": 0.02141041081225746, + "grad_norm": 0.577641487121582, + "kl": 0.10752715915441513, + "learning_rate": 1.7274575140626318e-06, + "loss": 0.0043, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 160 + }, + { + "completion_length": 125.83333587646484, + "epoch": 0.02154422587983407, + "grad_norm": 0.39314910769462585, + "kl": 0.04669243097305298, + "learning_rate": 1.6943357619237227e-06, + "loss": 0.0019, + "reward": 1.4166667461395264, + "reward_std": 1.2006943225860596, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 161 + }, + { + "completion_length": 129.0, + "epoch": 0.021678040947410677, + "grad_norm": 0.6266517639160156, + "kl": 0.07034413516521454, + "learning_rate": 1.661371075624363e-06, + "loss": 0.0028, + "reward": 1.6666667461395264, + "reward_std": 1.2909945249557495, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 162 + }, + { + "completion_length": 157.33334350585938, + "epoch": 0.021811856014987287, + "grad_norm": 0.6227591037750244, + "kl": 0.020313257351517677, + "learning_rate": 1.6285698816954626e-06, + "loss": 0.0008, + "reward": 1.3333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 163 + }, + { + "completion_length": 100.5, + "epoch": 0.021945671082563897, + "grad_norm": 0.724163830280304, + "kl": 0.07246337085962296, + "learning_rate": 1.5959385747947697e-06, + "loss": 0.0029, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 164 + }, + { + "completion_length": 195.83334350585938, + "epoch": 0.022079486150140507, + "grad_norm": 0.6425427198410034, + "kl": 0.009818270802497864, + "learning_rate": 1.56348351646022e-06, + "loss": 0.0004, + "reward": 0.5, + "reward_std": 1.0, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 165 + }, + { + "completion_length": 173.1666717529297, + "epoch": 0.022213301217717116, + "grad_norm": 0.31658393144607544, + "kl": 0.02298016846179962, + "learning_rate": 1.5312110338697427e-06, + "loss": 0.0009, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 166 + }, + { + "completion_length": 144.33334350585938, + "epoch": 0.022347116285293723, + "grad_norm": 0.7370114922523499, + "kl": 0.038862839341163635, + "learning_rate": 1.4991274186077632e-06, + "loss": 0.0016, + "reward": 1.1519999504089355, + "reward_std": 1.4199146032333374, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.09800000488758087, + "step": 167 + }, + { + "completion_length": 108.16667175292969, + "epoch": 0.022480931352870333, + "grad_norm": 1.0748318433761597, + "kl": 0.05299171060323715, + "learning_rate": 1.467238925438646e-06, + "loss": 0.0021, + "reward": 2.5209999084472656, + "reward_std": 0.051439255475997925, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.021000001579523087, + "step": 168 + }, + { + "completion_length": 91.83333587646484, + "epoch": 0.022614746420446943, + "grad_norm": 1.3673151731491089, + "kl": 0.07639364898204803, + "learning_rate": 1.4355517710873184e-06, + "loss": 0.0031, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 169 + }, + { + "completion_length": 135.5, + "epoch": 0.022748561488023553, + "grad_norm": 0.4482136368751526, + "kl": 0.02127678319811821, + "learning_rate": 1.4040721330273063e-06, + "loss": 0.0009, + "reward": 2.1666667461395264, + "reward_std": 0.8164965510368347, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 170 + }, + { + "completion_length": 165.33334350585938, + "epoch": 0.02288237655560016, + "grad_norm": 0.27974173426628113, + "kl": 0.01391599141061306, + "learning_rate": 1.3728061482764238e-06, + "loss": 0.0006, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 171 + }, + { + "completion_length": 176.33334350585938, + "epoch": 0.02301619162317677, + "grad_norm": 0.4582180082798004, + "kl": 0.016940653324127197, + "learning_rate": 1.3417599122003464e-06, + "loss": 0.0007, + "reward": 1.4166667461395264, + "reward_std": 1.2006943225860596, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 172 + }, + { + "completion_length": 200.0, + "epoch": 0.02315000669075338, + "grad_norm": 0.0036287466064095497, + "kl": 0.0012827565660700202, + "learning_rate": 1.3109394773243117e-06, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 173 + }, + { + "completion_length": 85.83333587646484, + "epoch": 0.02328382175832999, + "grad_norm": 0.021944554522633553, + "kl": 0.04209326207637787, + "learning_rate": 1.280350852153168e-06, + "loss": 0.0017, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 174 + }, + { + "completion_length": 108.0, + "epoch": 0.0234176368259066, + "grad_norm": 0.01814507693052292, + "kl": 0.04845254868268967, + "learning_rate": 1.2500000000000007e-06, + "loss": 0.0019, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 175 + }, + { + "completion_length": 169.0, + "epoch": 0.023551451893483205, + "grad_norm": 0.418618768453598, + "kl": 0.01748467981815338, + "learning_rate": 1.2198928378235717e-06, + "loss": 0.0007, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 176 + }, + { + "completion_length": 172.0, + "epoch": 0.023685266961059815, + "grad_norm": 0.33497563004493713, + "kl": 0.023299388587474823, + "learning_rate": 1.1900352350748026e-06, + "loss": 0.0009, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 177 + }, + { + "completion_length": 176.6666717529297, + "epoch": 0.023819082028636425, + "grad_norm": 0.629429817199707, + "kl": 0.022387957200407982, + "learning_rate": 1.160433012552508e-06, + "loss": 0.0009, + "reward": 0.8541666865348816, + "reward_std": 1.2757759094238281, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 178 + }, + { + "completion_length": 105.0, + "epoch": 0.023952897096213035, + "grad_norm": 0.6294716596603394, + "kl": 0.019054202362895012, + "learning_rate": 1.1310919412686248e-06, + "loss": 0.0008, + "reward": 2.1666667461395264, + "reward_std": 0.8164965510368347, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 179 + }, + { + "completion_length": 172.5, + "epoch": 0.02408671216378964, + "grad_norm": 0.318983793258667, + "kl": 0.022515472024679184, + "learning_rate": 1.1020177413231334e-06, + "loss": 0.0009, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 180 + }, + { + "completion_length": 126.33333587646484, + "epoch": 0.02422052723136625, + "grad_norm": 0.007083941251039505, + "kl": 0.026608020067214966, + "learning_rate": 1.073216080788921e-06, + "loss": 0.0011, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 181 + }, + { + "completion_length": 101.83333587646484, + "epoch": 0.02435434229894286, + "grad_norm": 0.01637137122452259, + "kl": 0.05336300656199455, + "learning_rate": 1.0446925746067768e-06, + "loss": 0.0021, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 182 + }, + { + "completion_length": 133.6666717529297, + "epoch": 0.02448815736651947, + "grad_norm": 0.010956374928355217, + "kl": 0.015068410895764828, + "learning_rate": 1.0164527834907468e-06, + "loss": 0.0006, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 183 + }, + { + "completion_length": 152.83334350585938, + "epoch": 0.02462197243409608, + "grad_norm": 0.059698257595300674, + "kl": 0.026588236913084984, + "learning_rate": 9.88502212844063e-07, + "loss": 0.0011, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 184 + }, + { + "completion_length": 104.5, + "epoch": 0.024755787501672687, + "grad_norm": 0.8568313121795654, + "kl": 0.1081070601940155, + "learning_rate": 9.608463116858544e-07, + "loss": 0.0043, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 185 + }, + { + "completion_length": 188.0, + "epoch": 0.024889602569249297, + "grad_norm": 0.4310242533683777, + "kl": 0.009734027087688446, + "learning_rate": 9.334904715888496e-07, + "loss": 0.0004, + "reward": 0.0833333358168602, + "reward_std": 0.20412413775920868, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 186 + }, + { + "completion_length": 134.1666717529297, + "epoch": 0.025023417636825907, + "grad_norm": 0.43237951397895813, + "kl": 0.11007413268089294, + "learning_rate": 9.064400256282757e-07, + "loss": 0.0044, + "reward": 2.4739999771118164, + "reward_std": 0.063686802983284, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.026000000536441803, + "step": 187 + }, + { + "completion_length": 120.83333587646484, + "epoch": 0.025157232704402517, + "grad_norm": 0.5578576922416687, + "kl": 0.06421395391225815, + "learning_rate": 8.797002473421729e-07, + "loss": 0.0026, + "reward": 1.3333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 188 + }, + { + "completion_length": 129.33334350585938, + "epoch": 0.025291047771979124, + "grad_norm": 0.010872965678572655, + "kl": 0.04434513673186302, + "learning_rate": 8.532763497032987e-07, + "loss": 0.0018, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 189 + }, + { + "completion_length": 82.33333587646484, + "epoch": 0.025424862839555733, + "grad_norm": 0.8208261728286743, + "kl": 0.12054745852947235, + "learning_rate": 8.271734841028553e-07, + "loss": 0.0048, + "reward": 2.125, + "reward_std": 1.0458250045776367, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0416666679084301, + "step": 190 + }, + { + "completion_length": 157.6666717529297, + "epoch": 0.025558677907132343, + "grad_norm": 0.007430327590554953, + "kl": 0.0287347212433815, + "learning_rate": 8.013967393462094e-07, + "loss": 0.0011, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 191 + }, + { + "completion_length": 105.33333587646484, + "epoch": 0.025692492974708953, + "grad_norm": 0.5002444386482239, + "kl": 0.05566655844449997, + "learning_rate": 7.759511406608255e-07, + "loss": 0.0022, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 192 + }, + { + "completion_length": 135.83334350585938, + "epoch": 0.02582630804228556, + "grad_norm": 0.663666844367981, + "kl": 0.07299192249774933, + "learning_rate": 7.508416487165862e-07, + "loss": 0.0029, + "reward": 1.68149995803833, + "reward_std": 1.2683271169662476, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.014833333902060986, + "step": 193 + }, + { + "completion_length": 153.5, + "epoch": 0.02596012310986217, + "grad_norm": 0.48523613810539246, + "kl": 0.019618486985564232, + "learning_rate": 7.260731586586983e-07, + "loss": 0.0008, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 194 + }, + { + "completion_length": 185.1666717529297, + "epoch": 0.02609393817743878, + "grad_norm": 0.48822808265686035, + "kl": 0.01835816167294979, + "learning_rate": 7.016504991533727e-07, + "loss": 0.0007, + "reward": 0.1666666716337204, + "reward_std": 0.25819891691207886, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 195 + }, + { + "completion_length": 98.16667175292969, + "epoch": 0.02622775324501539, + "grad_norm": 0.8225710391998291, + "kl": 0.05286232382059097, + "learning_rate": 6.775784314464717e-07, + "loss": 0.0021, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 196 + }, + { + "completion_length": 95.0, + "epoch": 0.026361568312592, + "grad_norm": 0.02045668661594391, + "kl": 0.11426003277301788, + "learning_rate": 6.538616484352902e-07, + "loss": 0.0046, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 197 + }, + { + "completion_length": 161.1666717529297, + "epoch": 0.026495383380168606, + "grad_norm": 0.5276882648468018, + "kl": 0.02128939889371395, + "learning_rate": 6.305047737536707e-07, + "loss": 0.0009, + "reward": 0.6041666865348816, + "reward_std": 0.9566108584403992, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 198 + }, + { + "completion_length": 109.33333587646484, + "epoch": 0.026629198447745216, + "grad_norm": 0.009646751917898655, + "kl": 0.032662972807884216, + "learning_rate": 6.075123608706093e-07, + "loss": 0.0013, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 199 + }, + { + "completion_length": 183.5, + "epoch": 0.026763013515321826, + "grad_norm": 0.4662231504917145, + "kl": 0.018144948408007622, + "learning_rate": 5.848888922025553e-07, + "loss": 0.0007, + "reward": 0.8541666865348816, + "reward_std": 1.2757759094238281, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.02083333395421505, + "step": 200 + }, + { + "completion_length": 134.33334350585938, + "epoch": 0.026896828582898435, + "grad_norm": 0.3459940552711487, + "kl": 0.02849223092198372, + "learning_rate": 5.626387782395512e-07, + "loss": 0.0011, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 201 + }, + { + "completion_length": 144.5, + "epoch": 0.027030643650475042, + "grad_norm": 0.4389075040817261, + "kl": 0.10118179768323898, + "learning_rate": 5.407663566854008e-07, + "loss": 0.004, + "reward": 0.8068333864212036, + "reward_std": 1.312966227531433, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.026499999687075615, + "step": 202 + }, + { + "completion_length": 135.1666717529297, + "epoch": 0.027164458718051652, + "grad_norm": 0.3375273644924164, + "kl": 0.03832804784178734, + "learning_rate": 5.192758916120236e-07, + "loss": 0.0015, + "reward": 1.5, + "reward_std": 1.095445156097412, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 203 + }, + { + "completion_length": 149.6666717529297, + "epoch": 0.02729827378562826, + "grad_norm": 0.018328191712498665, + "kl": 0.03014025092124939, + "learning_rate": 4.981715726281666e-07, + "loss": 0.0012, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 204 + }, + { + "completion_length": 160.1666717529297, + "epoch": 0.02743208885320487, + "grad_norm": 0.08511482924222946, + "kl": 0.06747082620859146, + "learning_rate": 4.774575140626317e-07, + "loss": 0.0027, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 205 + }, + { + "completion_length": 82.66667175292969, + "epoch": 0.02756590392078148, + "grad_norm": 4.189925193786621, + "kl": 0.9347108602523804, + "learning_rate": 4.5713775416217884e-07, + "loss": 0.0374, + "reward": 2.1038334369659424, + "reward_std": 0.9704062342643738, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.020500000566244125, + "step": 206 + }, + { + "completion_length": 195.83334350585938, + "epoch": 0.027699718988358088, + "grad_norm": 0.019951414316892624, + "kl": 0.005952201783657074, + "learning_rate": 4.372162543042624e-07, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 207 + }, + { + "completion_length": 81.33333587646484, + "epoch": 0.027833534055934698, + "grad_norm": 0.6730319261550903, + "kl": 0.09711631387472153, + "learning_rate": 4.1769689822475147e-07, + "loss": 0.0039, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 208 + }, + { + "completion_length": 148.83334350585938, + "epoch": 0.027967349123511308, + "grad_norm": 0.012037621811032295, + "kl": 0.03848704323172569, + "learning_rate": 3.9858349126078945e-07, + "loss": 0.0015, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 209 + }, + { + "completion_length": 140.0, + "epoch": 0.028101164191087918, + "grad_norm": 0.9327221512794495, + "kl": 0.03975226730108261, + "learning_rate": 3.798797596089351e-07, + "loss": 0.0016, + "reward": 2.1533334255218506, + "reward_std": 0.8105965852737427, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.013333333656191826, + "step": 210 + }, + { + "completion_length": 138.5, + "epoch": 0.028234979258664524, + "grad_norm": 0.3639069199562073, + "kl": 0.019840382039546967, + "learning_rate": 3.615893495987335e-07, + "loss": 0.0008, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 211 + }, + { + "completion_length": 127.5, + "epoch": 0.028368794326241134, + "grad_norm": 1.152321457862854, + "kl": 0.12709830701351166, + "learning_rate": 3.4371582698185636e-07, + "loss": 0.0051, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 212 + }, + { + "completion_length": 111.66667175292969, + "epoch": 0.028502609393817744, + "grad_norm": 0.6623407602310181, + "kl": 0.03994090482592583, + "learning_rate": 3.262626762369525e-07, + "loss": 0.0016, + "reward": 2.5416667461395264, + "reward_std": 0.10206206887960434, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0416666679084301, + "step": 213 + }, + { + "completion_length": 200.0, + "epoch": 0.028636424461394354, + "grad_norm": 0.004210959654301405, + "kl": 0.0024131210520863533, + "learning_rate": 3.092332998903416e-07, + "loss": 0.0001, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 214 + }, + { + "completion_length": 99.16667175292969, + "epoch": 0.028770239528970964, + "grad_norm": 0.014651302248239517, + "kl": 0.09786570072174072, + "learning_rate": 2.9263101785268253e-07, + "loss": 0.0039, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 215 + }, + { + "completion_length": 123.66667175292969, + "epoch": 0.02890405459654757, + "grad_norm": 0.006985802669078112, + "kl": 0.024645794183015823, + "learning_rate": 2.764590667717562e-07, + "loss": 0.001, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 216 + }, + { + "completion_length": 142.83334350585938, + "epoch": 0.02903786966412418, + "grad_norm": 0.5996547341346741, + "kl": 0.037068672478199005, + "learning_rate": 2.6072059940146775e-07, + "loss": 0.0015, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 217 + }, + { + "completion_length": 123.16667175292969, + "epoch": 0.02917168473170079, + "grad_norm": 0.005444398615509272, + "kl": 0.0173953827470541, + "learning_rate": 2.454186839872158e-07, + "loss": 0.0007, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 218 + }, + { + "completion_length": 119.33333587646484, + "epoch": 0.0293054997992774, + "grad_norm": 0.4850221574306488, + "kl": 0.04719596356153488, + "learning_rate": 2.3055630366772857e-07, + "loss": 0.0019, + "reward": 2.1666667461395264, + "reward_std": 0.8164965510368347, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 219 + }, + { + "completion_length": 194.83334350585938, + "epoch": 0.029439314866854006, + "grad_norm": 0.2741755247116089, + "kl": 0.005098981317132711, + "learning_rate": 2.1613635589349756e-07, + "loss": 0.0002, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 220 + }, + { + "completion_length": 180.0, + "epoch": 0.029573129934430616, + "grad_norm": 0.4658772051334381, + "kl": 0.02872592769563198, + "learning_rate": 2.0216165186191406e-07, + "loss": 0.0011, + "reward": 0.9166666865348816, + "reward_std": 1.2416388988494873, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 221 + }, + { + "completion_length": 180.5, + "epoch": 0.029706945002007226, + "grad_norm": 0.5886812210083008, + "kl": 0.026107415556907654, + "learning_rate": 1.8863491596921745e-07, + "loss": 0.001, + "reward": 0.8080000281333923, + "reward_std": 1.317206859588623, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.025333335623145103, + "step": 222 + }, + { + "completion_length": 137.33334350585938, + "epoch": 0.029840760069583836, + "grad_norm": 0.013461791910231113, + "kl": 0.020822227001190186, + "learning_rate": 1.7555878527937164e-07, + "loss": 0.0008, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 223 + }, + { + "completion_length": 88.83333587646484, + "epoch": 0.029974575137160443, + "grad_norm": 0.9285796284675598, + "kl": 0.061391860246658325, + "learning_rate": 1.629358090099639e-07, + "loss": 0.0025, + "reward": 2.1008334159851074, + "reward_std": 1.0300506353378296, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.017500000074505806, + "step": 224 + }, + { + "completion_length": 111.83333587646484, + "epoch": 0.030108390204737052, + "grad_norm": 0.47369858622550964, + "kl": 0.05458936095237732, + "learning_rate": 1.507684480352292e-07, + "loss": 0.0022, + "reward": 2.0833334922790527, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 225 + }, + { + "completion_length": 102.5, + "epoch": 0.030242205272313662, + "grad_norm": 0.5270379185676575, + "kl": 0.04939878731966019, + "learning_rate": 1.3905907440629752e-07, + "loss": 0.002, + "reward": 2.484499931335449, + "reward_std": 0.03796706721186638, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.015500000678002834, + "step": 226 + }, + { + "completion_length": 122.0, + "epoch": 0.030376020339890272, + "grad_norm": 1.011409878730774, + "kl": 0.07699307054281235, + "learning_rate": 1.278099708887587e-07, + "loss": 0.0031, + "reward": 2.0989999771118164, + "reward_std": 0.9822453856468201, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.015666667371988297, + "step": 227 + }, + { + "completion_length": 177.0, + "epoch": 0.030509835407466882, + "grad_norm": 0.24471516907215118, + "kl": 0.013278186321258545, + "learning_rate": 1.1702333051763271e-07, + "loss": 0.0005, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 228 + }, + { + "completion_length": 125.66667175292969, + "epoch": 0.03064365047504349, + "grad_norm": 0.4674018323421478, + "kl": 0.03539639338850975, + "learning_rate": 1.067012561698319e-07, + "loss": 0.0014, + "reward": 1.6666667461395264, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 229 + }, + { + "completion_length": 136.1666717529297, + "epoch": 0.0307774655426201, + "grad_norm": 0.5469502806663513, + "kl": 0.013292806223034859, + "learning_rate": 9.684576015420277e-08, + "loss": 0.0005, + "reward": 0.75, + "reward_std": 0.8803408145904541, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 230 + }, + { + "completion_length": 188.1666717529297, + "epoch": 0.03091128061019671, + "grad_norm": 0.48473238945007324, + "kl": 0.009322606027126312, + "learning_rate": 8.745876381922147e-08, + "loss": 0.0004, + "reward": 0.4166666865348816, + "reward_std": 1.0206208229064941, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 231 + }, + { + "completion_length": 162.0, + "epoch": 0.03104509567777332, + "grad_norm": 0.008338610641658306, + "kl": 0.05397571250796318, + "learning_rate": 7.854209717842231e-08, + "loss": 0.0022, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 232 + }, + { + "completion_length": 109.16667175292969, + "epoch": 0.031178910745349925, + "grad_norm": 0.05844065919518471, + "kl": 0.04213399440050125, + "learning_rate": 7.009749855363457e-08, + "loss": 0.0017, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 233 + }, + { + "completion_length": 172.6666717529297, + "epoch": 0.03131272581292654, + "grad_norm": 0.36787256598472595, + "kl": 0.02669469267129898, + "learning_rate": 6.212661423609184e-08, + "loss": 0.0011, + "reward": 1.75, + "reward_std": 1.172603964805603, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 234 + }, + { + "completion_length": 137.1666717529297, + "epoch": 0.031446540880503145, + "grad_norm": 0.9061893820762634, + "kl": 0.04043171927332878, + "learning_rate": 5.463099816548578e-08, + "loss": 0.0016, + "reward": 1.6258333921432495, + "reward_std": 1.356467604637146, + "rewards/correctness_reward_func": 1.3333333730697632, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.04083333536982536, + "step": 235 + }, + { + "completion_length": 128.5, + "epoch": 0.03158035594807975, + "grad_norm": 0.8342092037200928, + "kl": 0.051375072449445724, + "learning_rate": 4.761211162702117e-08, + "loss": 0.0021, + "reward": 1.25, + "reward_std": 1.3693064451217651, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 236 + }, + { + "completion_length": 192.1666717529297, + "epoch": 0.031714171015656364, + "grad_norm": 0.004815606400370598, + "kl": 0.005360649898648262, + "learning_rate": 4.1071322966535487e-08, + "loss": 0.0002, + "reward": 0.0, + "reward_std": 0.0, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 237 + }, + { + "completion_length": 162.33334350585938, + "epoch": 0.03184798608323297, + "grad_norm": 0.40393784642219543, + "kl": 0.02690042555332184, + "learning_rate": 3.5009907323737826e-08, + "loss": 0.0011, + "reward": 2.052999973297119, + "reward_std": 1.0083918571472168, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03033333271741867, + "step": 238 + }, + { + "completion_length": 150.33334350585938, + "epoch": 0.031981801150809584, + "grad_norm": 0.5924004912376404, + "kl": 0.0438881441950798, + "learning_rate": 2.9429046383618042e-08, + "loss": 0.0018, + "reward": 1.9803333282470703, + "reward_std": 0.9830150008201599, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.4166666865348816, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.10300000011920929, + "step": 239 + }, + { + "completion_length": 185.5, + "epoch": 0.03211561621838619, + "grad_norm": 0.4823758602142334, + "kl": 0.01092308945953846, + "learning_rate": 2.4329828146074096e-08, + "loss": 0.0004, + "reward": 0.0833333358168602, + "reward_std": 0.20412413775920868, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0833333358168602, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 240 + }, + { + "completion_length": 199.33334350585938, + "epoch": 0.0322494312859628, + "grad_norm": 0.9771707057952881, + "kl": 0.11909093707799911, + "learning_rate": 1.9713246713805588e-08, + "loss": 0.0048, + "reward": 0.4271666705608368, + "reward_std": 0.8259710073471069, + "rewards/correctness_reward_func": 0.3333333432674408, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.0728333368897438, + "step": 241 + }, + { + "completion_length": 142.83334350585938, + "epoch": 0.03238324635353941, + "grad_norm": 0.013215508311986923, + "kl": 0.019352126866579056, + "learning_rate": 1.5580202098509078e-08, + "loss": 0.0008, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 242 + }, + { + "completion_length": 159.33334350585938, + "epoch": 0.03251706142111602, + "grad_norm": 0.7379459738731384, + "kl": 0.055626656860113144, + "learning_rate": 1.193150004542204e-08, + "loss": 0.0022, + "reward": -0.10683333873748779, + "reward_std": 0.13728278875350952, + "rewards/correctness_reward_func": 0.0, + "rewards/int_reward_func": 0.0, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.10683333873748779, + "step": 243 + }, + { + "completion_length": 123.16667175292969, + "epoch": 0.03265087648869262, + "grad_norm": 0.008916381746530533, + "kl": 0.02974400669336319, + "learning_rate": 8.767851876239075e-09, + "loss": 0.0012, + "reward": 2.5, + "reward_std": 0.0, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 244 + }, + { + "completion_length": 126.16667175292969, + "epoch": 0.03278469155626924, + "grad_norm": 0.8518889546394348, + "kl": 0.02666613459587097, + "learning_rate": 6.089874350439507e-09, + "loss": 0.0011, + "reward": 1.245500087738037, + "reward_std": 1.3644126653671265, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.25, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.004500000271946192, + "step": 245 + }, + { + "completion_length": 112.5, + "epoch": 0.03291850662384584, + "grad_norm": 0.6925764083862305, + "kl": 0.07085268944501877, + "learning_rate": 3.8980895450474455e-09, + "loss": 0.0028, + "reward": 2.454166889190674, + "reward_std": 0.11226829886436462, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.04583333432674408, + "step": 246 + }, + { + "completion_length": 106.83333587646484, + "epoch": 0.03305232169142246, + "grad_norm": 0.5025346279144287, + "kl": 0.05391158163547516, + "learning_rate": 2.192924752854042e-09, + "loss": 0.0022, + "reward": 2.4696667194366455, + "reward_std": 0.07430125772953033, + "rewards/correctness_reward_func": 2.0, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": -0.03033333271741867, + "step": 247 + }, + { + "completion_length": 103.0, + "epoch": 0.03318613675899906, + "grad_norm": 0.6465145945549011, + "kl": 0.08165942132472992, + "learning_rate": 9.747123991141193e-10, + "loss": 0.0033, + "reward": 2.1666667461395264, + "reward_std": 0.8164965510368347, + "rewards/correctness_reward_func": 1.6666667461395264, + "rewards/int_reward_func": 0.5, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 248 + }, + { + "completion_length": 181.6666717529297, + "epoch": 0.03331995182657567, + "grad_norm": 0.398413747549057, + "kl": 0.016869667917490005, + "learning_rate": 2.43689976739403e-10, + "loss": 0.0007, + "reward": 0.8333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 0.6666666865348816, + "rewards/int_reward_func": 0.1666666716337204, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 249 + }, + { + "completion_length": 176.33334350585938, + "epoch": 0.03345376689415228, + "grad_norm": 0.4862636625766754, + "kl": 0.020642109215259552, + "learning_rate": 0.0, + "loss": 0.0008, + "reward": 1.3333333730697632, + "reward_std": 1.29099440574646, + "rewards/correctness_reward_func": 1.0, + "rewards/int_reward_func": 0.3333333432674408, + "rewards/soft_format_reward_func": 0.0, + "rewards/strict_format_reward_func": 0.0, + "rewards/xmlcount_reward_func": 0.0, + "step": 250 + } + ], + "logging_steps": 1, + "max_steps": 250, + "num_input_tokens_seen": 0, + "num_train_epochs": 1, + "save_steps": 250, + "stateful_callbacks": { + "TrainerControl": { + "args": { + "should_epoch_stop": false, + "should_evaluate": false, + "should_log": false, + "should_save": true, + "should_training_stop": true + }, + "attributes": {} + } + }, + "total_flos": 0.0, + "train_batch_size": 1, + "trial_name": null, + "trial_params": null +}