| { | |
| "best_global_step": null, | |
| "best_metric": null, | |
| "best_model_checkpoint": null, | |
| "epoch": 0.22857142857142856, | |
| "eval_steps": 500, | |
| "global_step": 200, | |
| "is_hyper_param_search": false, | |
| "is_local_process_zero": true, | |
| "is_world_process_zero": true, | |
| "log_history": [ | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2571.2083587646484, | |
| "epoch": 0.001142857142857143, | |
| "grad_norm": 0.25632715225219727, | |
| "kl": 0.0, | |
| "learning_rate": 0.0, | |
| "loss": 0.1082, | |
| "reward": 0.21363236638717353, | |
| "reward_std": 0.5417899899184704, | |
| "rewards/accuracy_reward": 0.22916667349636555, | |
| "rewards/cosine_scaled_reward": -0.015534311532974243, | |
| "step": 1 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2804.395881652832, | |
| "epoch": 0.002285714285714286, | |
| "grad_norm": 0.2158345729112625, | |
| "kl": 0.0, | |
| "learning_rate": 5e-08, | |
| "loss": 0.0476, | |
| "reward": 0.179365461692214, | |
| "reward_std": 0.24326776154339314, | |
| "rewards/accuracy_reward": 0.2291666716337204, | |
| "rewards/cosine_scaled_reward": -0.04980122856795788, | |
| "step": 2 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3340.9791717529297, | |
| "epoch": 0.0034285714285714284, | |
| "grad_norm": 0.1626892238855362, | |
| "kl": 4.589557647705078e-05, | |
| "learning_rate": 1e-07, | |
| "loss": -0.0326, | |
| "reward": -0.18680910766124725, | |
| "reward_std": 0.2629435881972313, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.2076424416154623, | |
| "step": 3 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2383.8125228881836, | |
| "epoch": 0.004571428571428572, | |
| "grad_norm": 0.321321964263916, | |
| "kl": 3.878772258758545e-05, | |
| "learning_rate": 1.5e-07, | |
| "loss": 0.0713, | |
| "reward": 0.037019677460193634, | |
| "reward_std": 0.5308115221560001, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.10881366301327944, | |
| "step": 4 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3285.6458740234375, | |
| "epoch": 0.005714285714285714, | |
| "grad_norm": 0.16392037272453308, | |
| "kl": 3.78340482711792e-05, | |
| "learning_rate": 2e-07, | |
| "loss": 0.0192, | |
| "reward": -0.10475460533052683, | |
| "reward_std": 0.4349679294973612, | |
| "rewards/accuracy_reward": 0.0833333358168602, | |
| "rewards/cosine_scaled_reward": -0.18808794301003218, | |
| "step": 5 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2898.625045776367, | |
| "epoch": 0.006857142857142857, | |
| "grad_norm": 0.19641205668449402, | |
| "kl": 4.819035530090332e-05, | |
| "learning_rate": 2.5e-07, | |
| "loss": 0.0638, | |
| "reward": -0.0739727895706892, | |
| "reward_std": 0.40896974317729473, | |
| "rewards/accuracy_reward": 0.0833333358168602, | |
| "rewards/cosine_scaled_reward": -0.15730612399056554, | |
| "step": 6 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3123.3333740234375, | |
| "epoch": 0.008, | |
| "grad_norm": 0.15763328969478607, | |
| "kl": 2.3871660232543945e-05, | |
| "learning_rate": 3e-07, | |
| "loss": 0.0023, | |
| "reward": 0.2681586788967252, | |
| "reward_std": 0.4782935921102762, | |
| "rewards/accuracy_reward": 0.27083333767950535, | |
| "rewards/cosine_scaled_reward": -0.0026746466755867004, | |
| "step": 7 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2718.1458435058594, | |
| "epoch": 0.009142857142857144, | |
| "grad_norm": 0.18417485058307648, | |
| "kl": 2.504885196685791e-05, | |
| "learning_rate": 3.5e-07, | |
| "loss": 0.0037, | |
| "reward": 0.43941989610902965, | |
| "reward_std": 0.6203235052525997, | |
| "rewards/accuracy_reward": 0.35416667349636555, | |
| "rewards/cosine_scaled_reward": 0.08525321818888187, | |
| "step": 8 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3126.979232788086, | |
| "epoch": 0.010285714285714285, | |
| "grad_norm": 0.1806861162185669, | |
| "kl": 4.266202449798584e-05, | |
| "learning_rate": 4e-07, | |
| "loss": 0.0192, | |
| "reward": 0.06813614349812269, | |
| "reward_std": 0.5027568060904741, | |
| "rewards/accuracy_reward": 0.1666666716337204, | |
| "rewards/cosine_scaled_reward": -0.09853052510879934, | |
| "step": 9 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2906.416717529297, | |
| "epoch": 0.011428571428571429, | |
| "grad_norm": 0.18246354162693024, | |
| "kl": 2.8778158593922853e-05, | |
| "learning_rate": 4.5e-07, | |
| "loss": 0.0043, | |
| "reward": -0.07218722999095917, | |
| "reward_std": 0.47773855179548264, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/cosine_scaled_reward": -0.17635390162467957, | |
| "step": 10 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3303.500030517578, | |
| "epoch": 0.012571428571428572, | |
| "grad_norm": 0.15648911893367767, | |
| "kl": 3.5509467124938965e-05, | |
| "learning_rate": 5e-07, | |
| "loss": -0.0166, | |
| "reward": -0.156071281991899, | |
| "reward_std": 0.2669069590047002, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.1977379499003291, | |
| "step": 11 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2357.5208740234375, | |
| "epoch": 0.013714285714285714, | |
| "grad_norm": 0.2004307210445404, | |
| "kl": 3.539770841598511e-05, | |
| "learning_rate": 5.5e-07, | |
| "loss": 0.0214, | |
| "reward": -0.02095029642805457, | |
| "reward_std": 0.5913799963891506, | |
| "rewards/accuracy_reward": 0.1458333358168602, | |
| "rewards/cosine_scaled_reward": -0.1667836436536163, | |
| "step": 12 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2870.541702270508, | |
| "epoch": 0.014857142857142857, | |
| "grad_norm": 0.18646906316280365, | |
| "kl": 3.197789192199707e-05, | |
| "learning_rate": 6e-07, | |
| "loss": -0.0017, | |
| "reward": 0.14097319915890694, | |
| "reward_std": 0.35468385741114616, | |
| "rewards/accuracy_reward": 0.1875000074505806, | |
| "rewards/cosine_scaled_reward": -0.04652681015431881, | |
| "step": 13 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2858.375015258789, | |
| "epoch": 0.016, | |
| "grad_norm": 0.2710908055305481, | |
| "kl": 2.9653310775756836e-05, | |
| "learning_rate": 6.5e-07, | |
| "loss": 0.063, | |
| "reward": 0.18795185792259872, | |
| "reward_std": 0.5712584815919399, | |
| "rewards/accuracy_reward": 0.20833334140479565, | |
| "rewards/cosine_scaled_reward": -0.0203814753331244, | |
| "step": 14 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2703.979202270508, | |
| "epoch": 0.017142857142857144, | |
| "grad_norm": 0.17628003656864166, | |
| "kl": 2.5141984224319458e-05, | |
| "learning_rate": 7e-07, | |
| "loss": -0.0072, | |
| "reward": 0.27426825650036335, | |
| "reward_std": 0.38823096454143524, | |
| "rewards/accuracy_reward": 0.2500000037252903, | |
| "rewards/cosine_scaled_reward": 0.02426825277507305, | |
| "step": 15 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3517.666717529297, | |
| "epoch": 0.018285714285714287, | |
| "grad_norm": 0.15789569914340973, | |
| "kl": 3.6329030990600586e-05, | |
| "learning_rate": 7.5e-07, | |
| "loss": -0.0084, | |
| "reward": -0.18198822857812047, | |
| "reward_std": 0.3138104174286127, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.22365489602088928, | |
| "step": 16 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2283.0417137145996, | |
| "epoch": 0.019428571428571427, | |
| "grad_norm": 0.2678299844264984, | |
| "kl": 3.6092475056648254e-05, | |
| "learning_rate": 8e-07, | |
| "loss": 0.0346, | |
| "reward": 0.19116858765482903, | |
| "reward_std": 0.6768512222915888, | |
| "rewards/accuracy_reward": 0.2291666716337204, | |
| "rewards/cosine_scaled_reward": -0.03799809608608484, | |
| "step": 17 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2891.6250534057617, | |
| "epoch": 0.02057142857142857, | |
| "grad_norm": 0.1524839550256729, | |
| "kl": 9.838491678237915e-06, | |
| "learning_rate": 8.499999999999999e-07, | |
| "loss": 0.0422, | |
| "reward": 0.019560502842068672, | |
| "reward_std": 0.42486423440277576, | |
| "rewards/accuracy_reward": 0.1250000037252903, | |
| "rewards/cosine_scaled_reward": -0.10543949622660875, | |
| "step": 18 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2730.6875381469727, | |
| "epoch": 0.021714285714285714, | |
| "grad_norm": 0.1984628587961197, | |
| "kl": 2.9616057872772217e-05, | |
| "learning_rate": 9e-07, | |
| "loss": 0.0244, | |
| "reward": 0.5108187086880207, | |
| "reward_std": 0.5117705333977938, | |
| "rewards/accuracy_reward": 0.3750000111758709, | |
| "rewards/cosine_scaled_reward": 0.1358186900615692, | |
| "step": 19 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2329.9583892822266, | |
| "epoch": 0.022857142857142857, | |
| "grad_norm": 0.29129523038864136, | |
| "kl": 1.1567026376724243e-05, | |
| "learning_rate": 9.499999999999999e-07, | |
| "loss": 0.1144, | |
| "reward": 0.3239101804792881, | |
| "reward_std": 0.5062661934643984, | |
| "rewards/accuracy_reward": 0.29166666977107525, | |
| "rewards/cosine_scaled_reward": 0.03224351815879345, | |
| "step": 20 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2660.2083587646484, | |
| "epoch": 0.024, | |
| "grad_norm": 0.19655585289001465, | |
| "kl": 2.6446767151355743e-05, | |
| "learning_rate": 1e-06, | |
| "loss": 0.062, | |
| "reward": 0.05283654620870948, | |
| "reward_std": 0.39788419753313065, | |
| "rewards/accuracy_reward": 0.1458333395421505, | |
| "rewards/cosine_scaled_reward": -0.09299679077230394, | |
| "step": 21 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 1825.020896911621, | |
| "epoch": 0.025142857142857144, | |
| "grad_norm": 0.2982518970966339, | |
| "kl": 2.1491199731826782e-05, | |
| "learning_rate": 9.99931462820376e-07, | |
| "loss": 0.0703, | |
| "reward": 0.23684986494481564, | |
| "reward_std": 0.4532635463401675, | |
| "rewards/accuracy_reward": 0.27083333767950535, | |
| "rewards/cosine_scaled_reward": -0.033983476692810655, | |
| "step": 22 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2637.2708892822266, | |
| "epoch": 0.026285714285714287, | |
| "grad_norm": 0.17908190190792084, | |
| "kl": 2.055056393146515e-05, | |
| "learning_rate": 9.997258721585931e-07, | |
| "loss": 0.0216, | |
| "reward": 0.04677679901942611, | |
| "reward_std": 0.5608484419062734, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.1198898684233427, | |
| "step": 23 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2836.437530517578, | |
| "epoch": 0.027428571428571427, | |
| "grad_norm": 0.18375183641910553, | |
| "kl": 2.3286789655685425e-05, | |
| "learning_rate": 9.993832906395582e-07, | |
| "loss": 0.0363, | |
| "reward": 0.1852389182895422, | |
| "reward_std": 0.5924705881625414, | |
| "rewards/accuracy_reward": 0.22916667349636555, | |
| "rewards/cosine_scaled_reward": -0.04392773995641619, | |
| "step": 24 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2892.875045776367, | |
| "epoch": 0.02857142857142857, | |
| "grad_norm": 0.19979171454906464, | |
| "kl": 3.429921343922615e-05, | |
| "learning_rate": 9.989038226169207e-07, | |
| "loss": -0.0319, | |
| "reward": 0.18543785763904452, | |
| "reward_std": 0.5226433798670769, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/cosine_scaled_reward": -0.022895504254847765, | |
| "step": 25 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2954.4791870117188, | |
| "epoch": 0.029714285714285714, | |
| "grad_norm": 0.1611883044242859, | |
| "kl": 2.4512410163879395e-05, | |
| "learning_rate": 9.982876141412855e-07, | |
| "loss": -0.0069, | |
| "reward": 0.2805785769596696, | |
| "reward_std": 0.33157524187117815, | |
| "rewards/accuracy_reward": 0.25000000558793545, | |
| "rewards/cosine_scaled_reward": 0.030578549951314926, | |
| "step": 26 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3068.104232788086, | |
| "epoch": 0.030857142857142857, | |
| "grad_norm": 0.1950128972530365, | |
| "kl": 8.122995495796204e-06, | |
| "learning_rate": 9.975348529157229e-07, | |
| "loss": 0.0453, | |
| "reward": 0.07182890921831131, | |
| "reward_std": 0.5058779399842024, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.09483776055276394, | |
| "step": 27 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2946.7083587646484, | |
| "epoch": 0.032, | |
| "grad_norm": 0.17087508738040924, | |
| "kl": 1.4003482647240162e-05, | |
| "learning_rate": 9.96645768238595e-07, | |
| "loss": -0.041, | |
| "reward": 0.18860503658652306, | |
| "reward_std": 0.5205447860062122, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/cosine_scaled_reward": -0.01972830155864358, | |
| "step": 28 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3220.2708740234375, | |
| "epoch": 0.03314285714285714, | |
| "grad_norm": 0.24361945688724518, | |
| "kl": 3.1463801860809326e-05, | |
| "learning_rate": 9.956206309337066e-07, | |
| "loss": -0.1063, | |
| "reward": -0.17443929053843021, | |
| "reward_std": 0.3365236781537533, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.2161059557693079, | |
| "step": 29 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2893.104248046875, | |
| "epoch": 0.03428571428571429, | |
| "grad_norm": 0.2155693769454956, | |
| "kl": 5.615316331386566e-05, | |
| "learning_rate": 9.944597532678119e-07, | |
| "loss": 0.0487, | |
| "reward": 0.27425617165863514, | |
| "reward_std": 0.6831754259765148, | |
| "rewards/accuracy_reward": 0.25000000558793545, | |
| "rewards/cosine_scaled_reward": 0.024256166070699692, | |
| "step": 30 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3071.4375, | |
| "epoch": 0.03542857142857143, | |
| "grad_norm": 0.17095845937728882, | |
| "kl": 4.3095555156469345e-05, | |
| "learning_rate": 9.931634888554935e-07, | |
| "loss": -0.0257, | |
| "reward": -0.04954687878489494, | |
| "reward_std": 0.3689954075962305, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/cosine_scaled_reward": -0.15371354296803474, | |
| "step": 31 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3270.375030517578, | |
| "epoch": 0.036571428571428574, | |
| "grad_norm": 0.1867983490228653, | |
| "kl": 4.005682421848178e-05, | |
| "learning_rate": 9.917322325514487e-07, | |
| "loss": -0.0114, | |
| "reward": 0.33223521208856255, | |
| "reward_std": 0.5476495027542114, | |
| "rewards/accuracy_reward": 0.2916666716337204, | |
| "rewards/cosine_scaled_reward": 0.04056852776557207, | |
| "step": 32 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3300.2708740234375, | |
| "epoch": 0.037714285714285714, | |
| "grad_norm": 0.15105699002742767, | |
| "kl": 0.00013676844537258148, | |
| "learning_rate": 9.901664203302124e-07, | |
| "loss": 0.0197, | |
| "reward": 0.19714780524373055, | |
| "reward_std": 0.5669918619096279, | |
| "rewards/accuracy_reward": 0.22916667349636555, | |
| "rewards/cosine_scaled_reward": -0.032018861966207623, | |
| "step": 33 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2588.1875610351562, | |
| "epoch": 0.038857142857142854, | |
| "grad_norm": 0.47045713663101196, | |
| "kl": 0.00015014410018920898, | |
| "learning_rate": 9.88466529153356e-07, | |
| "loss": 0.1528, | |
| "reward": 0.3600555928424001, | |
| "reward_std": 0.6696161534637213, | |
| "rewards/accuracy_reward": 0.31250000931322575, | |
| "rewards/cosine_scaled_reward": 0.0475555956363678, | |
| "step": 34 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2979.000030517578, | |
| "epoch": 0.04, | |
| "grad_norm": 0.21019528806209564, | |
| "kl": 0.00010133162140846252, | |
| "learning_rate": 9.866330768241983e-07, | |
| "loss": 0.0436, | |
| "reward": 0.08721516001969576, | |
| "reward_std": 0.6073270477354527, | |
| "rewards/accuracy_reward": 0.16666666977107525, | |
| "rewards/cosine_scaled_reward": -0.07945150946034119, | |
| "step": 35 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3349.1041870117188, | |
| "epoch": 0.04114285714285714, | |
| "grad_norm": 0.1755942404270172, | |
| "kl": 0.00011576339602470398, | |
| "learning_rate": 9.846666218300807e-07, | |
| "loss": 0.0059, | |
| "reward": -0.17237537261098623, | |
| "reward_std": 0.22124690702185035, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.19320870749652386, | |
| "step": 36 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3302.3333435058594, | |
| "epoch": 0.04228571428571429, | |
| "grad_norm": 0.172617107629776, | |
| "kl": 7.249228656291962e-05, | |
| "learning_rate": 9.825677631722435e-07, | |
| "loss": -0.0307, | |
| "reward": -0.2602706290781498, | |
| "reward_std": 0.16562421713024378, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/cosine_scaled_reward": -0.26027063466608524, | |
| "step": 37 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3232.187515258789, | |
| "epoch": 0.04342857142857143, | |
| "grad_norm": 0.16284486651420593, | |
| "kl": 8.244812488555908e-05, | |
| "learning_rate": 9.80337140183366e-07, | |
| "loss": -0.0182, | |
| "reward": -0.027252301573753357, | |
| "reward_std": 0.2725130654871464, | |
| "rewards/accuracy_reward": 0.1041666716337204, | |
| "rewards/cosine_scaled_reward": -0.13141895458102226, | |
| "step": 38 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2854.1042098999023, | |
| "epoch": 0.044571428571428574, | |
| "grad_norm": 0.17649812996387482, | |
| "kl": 9.210407733917236e-05, | |
| "learning_rate": 9.779754323328192e-07, | |
| "loss": 0.0249, | |
| "reward": 0.1576530784368515, | |
| "reward_std": 0.24235367169603705, | |
| "rewards/accuracy_reward": 0.2083333358168602, | |
| "rewards/cosine_scaled_reward": -0.05068026855587959, | |
| "step": 39 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2369.416702270508, | |
| "epoch": 0.045714285714285714, | |
| "grad_norm": 0.18827204406261444, | |
| "kl": 0.00034741777926683426, | |
| "learning_rate": 9.754833590196926e-07, | |
| "loss": -0.0217, | |
| "reward": 0.06249742116779089, | |
| "reward_std": 0.2527109682559967, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.0833359295502305, | |
| "step": 40 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2972.229217529297, | |
| "epoch": 0.046857142857142854, | |
| "grad_norm": 0.17488084733486176, | |
| "kl": 0.0001214444637298584, | |
| "learning_rate": 9.728616793536587e-07, | |
| "loss": -0.0016, | |
| "reward": 0.055548351956531405, | |
| "reward_std": 0.5645285593345761, | |
| "rewards/accuracy_reward": 0.16666666977107525, | |
| "rewards/cosine_scaled_reward": -0.11111831851303577, | |
| "step": 41 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2735.8958473205566, | |
| "epoch": 0.048, | |
| "grad_norm": 0.28289923071861267, | |
| "kl": 8.103251457214355e-05, | |
| "learning_rate": 9.701111919237408e-07, | |
| "loss": -0.0087, | |
| "reward": -0.2322952365502715, | |
| "reward_std": 0.2050471631810069, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.25312857422977686, | |
| "step": 42 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3113.8958587646484, | |
| "epoch": 0.04914285714285714, | |
| "grad_norm": 0.20076647400856018, | |
| "kl": 8.933991193771362e-05, | |
| "learning_rate": 9.672327345550543e-07, | |
| "loss": 0.0291, | |
| "reward": -0.03336653020232916, | |
| "reward_std": 0.36820667795836926, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/cosine_scaled_reward": -0.1375331999734044, | |
| "step": 43 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2842.6250228881836, | |
| "epoch": 0.05028571428571429, | |
| "grad_norm": 0.32278913259506226, | |
| "kl": 0.0004453342407941818, | |
| "learning_rate": 9.64227184053598e-07, | |
| "loss": 0.0449, | |
| "reward": 0.2544646831229329, | |
| "reward_std": 0.47283124178647995, | |
| "rewards/accuracy_reward": 0.2291666753590107, | |
| "rewards/cosine_scaled_reward": 0.025297993794083595, | |
| "step": 44 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3480.7708740234375, | |
| "epoch": 0.05142857142857143, | |
| "grad_norm": 0.1464184671640396, | |
| "kl": 0.00017410144209861755, | |
| "learning_rate": 9.610954559391704e-07, | |
| "loss": 0.0163, | |
| "reward": 0.20797571539878845, | |
| "reward_std": 0.4654997680336237, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/cosine_scaled_reward": -0.0003576315939426422, | |
| "step": 45 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3276.7916717529297, | |
| "epoch": 0.052571428571428575, | |
| "grad_norm": 0.20637758076190948, | |
| "kl": 0.0003883976023644209, | |
| "learning_rate": 9.578385041664925e-07, | |
| "loss": -0.0218, | |
| "reward": -0.2070446526631713, | |
| "reward_std": 0.15773496124893427, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/cosine_scaled_reward": -0.20704465080052614, | |
| "step": 46 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2968.0833740234375, | |
| "epoch": 0.053714285714285714, | |
| "grad_norm": 0.20068301260471344, | |
| "kl": 0.0002564918249845505, | |
| "learning_rate": 9.54457320834625e-07, | |
| "loss": 0.1738, | |
| "reward": 0.22800862789154053, | |
| "reward_std": 0.6421588193625212, | |
| "rewards/accuracy_reward": 0.2291666716337204, | |
| "rewards/cosine_scaled_reward": -0.0011580577120184898, | |
| "step": 47 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2857.354232788086, | |
| "epoch": 0.054857142857142854, | |
| "grad_norm": 0.20215612649917603, | |
| "kl": 0.0009041652083396912, | |
| "learning_rate": 9.509529358847654e-07, | |
| "loss": -0.0166, | |
| "reward": 0.11733034905046225, | |
| "reward_std": 0.36523033399134874, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.04933632561005652, | |
| "step": 48 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2516.229217529297, | |
| "epoch": 0.056, | |
| "grad_norm": 0.19533751904964447, | |
| "kl": 0.00025360286235809326, | |
| "learning_rate": 9.473264167865171e-07, | |
| "loss": -0.0101, | |
| "reward": 0.2620775690302253, | |
| "reward_std": 0.6330471076071262, | |
| "rewards/accuracy_reward": 0.2500000074505806, | |
| "rewards/cosine_scaled_reward": 0.012077568098902702, | |
| "step": 49 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2981.4375, | |
| "epoch": 0.05714285714285714, | |
| "grad_norm": 0.165849506855011, | |
| "kl": 0.00045928824692964554, | |
| "learning_rate": 9.43578868212728e-07, | |
| "loss": -0.0219, | |
| "reward": 0.291345676407218, | |
| "reward_std": 0.4359543416649103, | |
| "rewards/accuracy_reward": 0.29166667349636555, | |
| "rewards/cosine_scaled_reward": -0.0003209911519661546, | |
| "step": 50 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2388.0625381469727, | |
| "epoch": 0.05828571428571429, | |
| "grad_norm": 0.22810906171798706, | |
| "kl": 0.0011309236288070679, | |
| "learning_rate": 9.397114317029974e-07, | |
| "loss": 0.0198, | |
| "reward": 0.11073957197368145, | |
| "reward_std": 0.4677628371864557, | |
| "rewards/accuracy_reward": 0.1666666716337204, | |
| "rewards/cosine_scaled_reward": -0.055927105247974396, | |
| "step": 51 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2879.5417289733887, | |
| "epoch": 0.05942857142857143, | |
| "grad_norm": 0.22546225786209106, | |
| "kl": 0.0010811500251293182, | |
| "learning_rate": 9.357252853159505e-07, | |
| "loss": 0.0439, | |
| "reward": 0.2536798194050789, | |
| "reward_std": 0.6959168985486031, | |
| "rewards/accuracy_reward": 0.2708333395421505, | |
| "rewards/cosine_scaled_reward": -0.01715349592268467, | |
| "step": 52 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2816.3125610351562, | |
| "epoch": 0.060571428571428575, | |
| "grad_norm": 0.22971203923225403, | |
| "kl": 0.0007025115191936493, | |
| "learning_rate": 9.316216432703916e-07, | |
| "loss": 0.0842, | |
| "reward": 0.30444883555173874, | |
| "reward_std": 0.6236651353538036, | |
| "rewards/accuracy_reward": 0.2916666753590107, | |
| "rewards/cosine_scaled_reward": 0.01278215617639944, | |
| "step": 53 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2990.541732788086, | |
| "epoch": 0.061714285714285715, | |
| "grad_norm": 0.17176969349384308, | |
| "kl": 0.00022144615650177002, | |
| "learning_rate": 9.274017555754407e-07, | |
| "loss": 0.0564, | |
| "reward": 0.36173775792121887, | |
| "reward_std": 0.7157497424632311, | |
| "rewards/accuracy_reward": 0.31250000931322575, | |
| "rewards/cosine_scaled_reward": 0.04923774115741253, | |
| "step": 54 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3098.687530517578, | |
| "epoch": 0.06285714285714286, | |
| "grad_norm": 0.1664225459098816, | |
| "kl": 0.0008269157260656357, | |
| "learning_rate": 9.230669076497687e-07, | |
| "loss": 0.055, | |
| "reward": 0.33386562392115593, | |
| "reward_std": 0.4932790081948042, | |
| "rewards/accuracy_reward": 0.27083333395421505, | |
| "rewards/cosine_scaled_reward": 0.06303227692842484, | |
| "step": 55 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2980.1667098999023, | |
| "epoch": 0.064, | |
| "grad_norm": 0.22248487174510956, | |
| "kl": 0.0004000365734100342, | |
| "learning_rate": 9.186184199300463e-07, | |
| "loss": 0.0463, | |
| "reward": 0.1612101886421442, | |
| "reward_std": 0.3384838867932558, | |
| "rewards/accuracy_reward": 0.2083333432674408, | |
| "rewards/cosine_scaled_reward": -0.047123152762651443, | |
| "step": 56 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3345.5833740234375, | |
| "epoch": 0.06514285714285714, | |
| "grad_norm": 0.14005281031131744, | |
| "kl": 0.00021963752806186676, | |
| "learning_rate": 9.140576474687263e-07, | |
| "loss": 0.02, | |
| "reward": -0.10464224513270892, | |
| "reward_std": 0.44751227274537086, | |
| "rewards/accuracy_reward": 0.0833333358168602, | |
| "rewards/cosine_scaled_reward": -0.1879755798727274, | |
| "step": 57 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2249.7917098999023, | |
| "epoch": 0.06628571428571428, | |
| "grad_norm": 0.21027207374572754, | |
| "kl": 0.00269511342048645, | |
| "learning_rate": 9.093859795212817e-07, | |
| "loss": 0.0657, | |
| "reward": 0.3534491113387048, | |
| "reward_std": 0.5972686447203159, | |
| "rewards/accuracy_reward": 0.31250000186264515, | |
| "rewards/cosine_scaled_reward": 0.04094908758997917, | |
| "step": 58 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2903.854179382324, | |
| "epoch": 0.06742857142857143, | |
| "grad_norm": 0.205511212348938, | |
| "kl": 0.0010725334286689758, | |
| "learning_rate": 9.046048391230247e-07, | |
| "loss": -0.0834, | |
| "reward": -0.007946477271616459, | |
| "reward_std": 0.4581656437367201, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/cosine_scaled_reward": -0.13294648192822933, | |
| "step": 59 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3034.270866394043, | |
| "epoch": 0.06857142857142857, | |
| "grad_norm": 0.16402272880077362, | |
| "kl": 0.00045564770698547363, | |
| "learning_rate": 8.997156826556369e-07, | |
| "loss": 0.0219, | |
| "reward": -0.010321232955902815, | |
| "reward_std": 0.44436920061707497, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/cosine_scaled_reward": -0.13532123528420925, | |
| "step": 60 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3173.125030517578, | |
| "epoch": 0.06971428571428571, | |
| "grad_norm": 0.15932059288024902, | |
| "kl": 0.0008110031485557556, | |
| "learning_rate": 8.9471999940354e-07, | |
| "loss": 0.0541, | |
| "reward": -0.0845769364386797, | |
| "reward_std": 0.42896613012999296, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/cosine_scaled_reward": -0.20957693923264742, | |
| "step": 61 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2611.666717529297, | |
| "epoch": 0.07085714285714285, | |
| "grad_norm": 0.2158014327287674, | |
| "kl": 0.07185456156730652, | |
| "learning_rate": 8.896193111002475e-07, | |
| "loss": 0.0293, | |
| "reward": 0.34560922905802727, | |
| "reward_std": 0.7172879008576274, | |
| "rewards/accuracy_reward": 0.31250000558793545, | |
| "rewards/cosine_scaled_reward": 0.033109224401414394, | |
| "step": 62 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2483.729232788086, | |
| "epoch": 0.072, | |
| "grad_norm": 0.17524653673171997, | |
| "kl": 0.003895312547683716, | |
| "learning_rate": 8.844151714648274e-07, | |
| "loss": 0.0736, | |
| "reward": 0.3518992383033037, | |
| "reward_std": 0.4679460935294628, | |
| "rewards/accuracy_reward": 0.29166666977107525, | |
| "rewards/cosine_scaled_reward": 0.060232581570744514, | |
| "step": 63 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3119.6666870117188, | |
| "epoch": 0.07314285714285715, | |
| "grad_norm": 0.17840062081813812, | |
| "kl": 0.001272439956665039, | |
| "learning_rate": 8.791091657286267e-07, | |
| "loss": 0.0552, | |
| "reward": 0.021638838574290276, | |
| "reward_std": 0.5125578781589866, | |
| "rewards/accuracy_reward": 0.1458333358168602, | |
| "rewards/cosine_scaled_reward": -0.12419449165463448, | |
| "step": 64 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2688.958366394043, | |
| "epoch": 0.07428571428571429, | |
| "grad_norm": 0.1985544115304947, | |
| "kl": 0.001712799072265625, | |
| "learning_rate": 8.737029101523929e-07, | |
| "loss": -0.0062, | |
| "reward": 0.08414293639361858, | |
| "reward_std": 0.45870870165526867, | |
| "rewards/accuracy_reward": 0.1875000074505806, | |
| "rewards/cosine_scaled_reward": -0.10335706360638142, | |
| "step": 65 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2155.270839691162, | |
| "epoch": 0.07542857142857143, | |
| "grad_norm": 0.24202020466327667, | |
| "kl": 0.0014235973358154297, | |
| "learning_rate": 8.681980515339463e-07, | |
| "loss": 0.0342, | |
| "reward": 0.2718210807070136, | |
| "reward_std": 0.3149276631884277, | |
| "rewards/accuracy_reward": 0.2916666716337204, | |
| "rewards/cosine_scaled_reward": -0.019845602102577686, | |
| "step": 66 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3514.5, | |
| "epoch": 0.07657142857142857, | |
| "grad_norm": 0.13506484031677246, | |
| "kl": 0.0011570192873477936, | |
| "learning_rate": 8.625962667065487e-07, | |
| "loss": 0.0197, | |
| "reward": -0.17880290560424328, | |
| "reward_std": 0.3006857465952635, | |
| "rewards/accuracy_reward": 0.0625, | |
| "rewards/cosine_scaled_reward": -0.24130290746688843, | |
| "step": 67 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2224.1250228881836, | |
| "epoch": 0.07771428571428571, | |
| "grad_norm": 0.26422637701034546, | |
| "kl": 0.008173942565917969, | |
| "learning_rate": 8.568992620281243e-07, | |
| "loss": 0.1289, | |
| "reward": 0.26997855119407177, | |
| "reward_std": 0.5711956042796373, | |
| "rewards/accuracy_reward": 0.2500000074505806, | |
| "rewards/cosine_scaled_reward": 0.019978564232587814, | |
| "step": 68 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2771.104202270508, | |
| "epoch": 0.07885714285714286, | |
| "grad_norm": 0.22578439116477966, | |
| "kl": 0.004560351371765137, | |
| "learning_rate": 8.511087728614862e-07, | |
| "loss": -0.0186, | |
| "reward": -0.20487310830503702, | |
| "reward_std": 0.2483804766088724, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.22570644319057465, | |
| "step": 69 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3143.8333587646484, | |
| "epoch": 0.08, | |
| "grad_norm": 0.2783580422401428, | |
| "kl": 0.0023155808448791504, | |
| "learning_rate": 8.452265630457282e-07, | |
| "loss": 0.0232, | |
| "reward": -0.22550072893500328, | |
| "reward_std": 0.24843326956033707, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.24633406475186348, | |
| "step": 70 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2701.6666870117188, | |
| "epoch": 0.08114285714285714, | |
| "grad_norm": 0.24257808923721313, | |
| "kl": 0.0030676722526550293, | |
| "learning_rate": 8.392544243589427e-07, | |
| "loss": 0.0883, | |
| "reward": 0.10035606473684311, | |
| "reward_std": 0.5035490561276674, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.0663105882704258, | |
| "step": 71 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3351.541748046875, | |
| "epoch": 0.08228571428571428, | |
| "grad_norm": 0.1588461697101593, | |
| "kl": 0.00363922119140625, | |
| "learning_rate": 8.331941759724268e-07, | |
| "loss": 0.0452, | |
| "reward": -0.19743284268770367, | |
| "reward_std": 0.27725750021636486, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.2182661797851324, | |
| "step": 72 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3507.666717529297, | |
| "epoch": 0.08342857142857144, | |
| "grad_norm": 0.1444426029920578, | |
| "kl": 0.0006330609321594238, | |
| "learning_rate": 8.270476638965461e-07, | |
| "loss": 0.0331, | |
| "reward": -0.05398387461900711, | |
| "reward_std": 0.49269056133925915, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/cosine_scaled_reward": -0.15815055184066296, | |
| "step": 73 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3344.041717529297, | |
| "epoch": 0.08457142857142858, | |
| "grad_norm": 0.15516585111618042, | |
| "kl": 0.0021051764488220215, | |
| "learning_rate": 8.208167604184217e-07, | |
| "loss": 0.0095, | |
| "reward": 0.008867893368005753, | |
| "reward_std": 0.26112012285739183, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.1369654512964189, | |
| "step": 74 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2982.4583435058594, | |
| "epoch": 0.08571428571428572, | |
| "grad_norm": 0.13954143226146698, | |
| "kl": 0.011302351951599121, | |
| "learning_rate": 8.145033635316128e-07, | |
| "loss": 0.0333, | |
| "reward": 0.28953575156629086, | |
| "reward_std": 0.3247816953808069, | |
| "rewards/accuracy_reward": 0.2708333358168602, | |
| "rewards/cosine_scaled_reward": 0.018702420871704817, | |
| "step": 75 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3167.2083587646484, | |
| "epoch": 0.08685714285714285, | |
| "grad_norm": 0.1654420793056488, | |
| "kl": 0.0009011179208755493, | |
| "learning_rate": 8.081093963579707e-07, | |
| "loss": 0.014, | |
| "reward": -0.2234128573909402, | |
| "reward_std": 0.27190146408975124, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.24424619507044554, | |
| "step": 76 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3288.125030517578, | |
| "epoch": 0.088, | |
| "grad_norm": 0.15181006491184235, | |
| "kl": 0.0010012537240982056, | |
| "learning_rate": 8.01636806561836e-07, | |
| "loss": -0.013, | |
| "reward": 0.1649619173258543, | |
| "reward_std": 0.2921932004392147, | |
| "rewards/accuracy_reward": 0.1875, | |
| "rewards/cosine_scaled_reward": -0.022538071498274803, | |
| "step": 77 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3191.187530517578, | |
| "epoch": 0.08914285714285715, | |
| "grad_norm": 0.1391523778438568, | |
| "kl": 0.0008662641048431396, | |
| "learning_rate": 7.950875657567621e-07, | |
| "loss": 0.0498, | |
| "reward": 0.156300887465477, | |
| "reward_std": 0.28907839488238096, | |
| "rewards/accuracy_reward": 0.1875, | |
| "rewards/cosine_scaled_reward": -0.031199105083942413, | |
| "step": 78 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2525.6459159851074, | |
| "epoch": 0.09028571428571429, | |
| "grad_norm": 0.22027799487113953, | |
| "kl": 0.0032279491424560547, | |
| "learning_rate": 7.884636689049422e-07, | |
| "loss": -0.0856, | |
| "reward": 0.27039410918951035, | |
| "reward_std": 0.4748187121003866, | |
| "rewards/accuracy_reward": 0.29166666977107525, | |
| "rewards/cosine_scaled_reward": -0.021272567100822926, | |
| "step": 79 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3310.000030517578, | |
| "epoch": 0.09142857142857143, | |
| "grad_norm": 0.15355749428272247, | |
| "kl": 0.0017953217029571533, | |
| "learning_rate": 7.817671337095244e-07, | |
| "loss": 0.0254, | |
| "reward": 0.15592915751039982, | |
| "reward_std": 0.45574192702770233, | |
| "rewards/accuracy_reward": 0.20833334140479565, | |
| "rewards/cosine_scaled_reward": -0.05240418389439583, | |
| "step": 80 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3161.375030517578, | |
| "epoch": 0.09257142857142857, | |
| "grad_norm": 0.20919400453567505, | |
| "kl": 0.004012107849121094, | |
| "learning_rate": 7.75e-07, | |
| "loss": 0.0254, | |
| "reward": -0.12468860670924187, | |
| "reward_std": 0.35988178849220276, | |
| "rewards/accuracy_reward": 0.06250000186264515, | |
| "rewards/cosine_scaled_reward": -0.1871886090375483, | |
| "step": 81 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2771.8958435058594, | |
| "epoch": 0.09371428571428571, | |
| "grad_norm": 0.184526726603508, | |
| "kl": 0.0013185292482376099, | |
| "learning_rate": 7.681643291108517e-07, | |
| "loss": 0.0203, | |
| "reward": 0.25131775764748454, | |
| "reward_std": 0.3994978480041027, | |
| "rewards/accuracy_reward": 0.2500000037252903, | |
| "rewards/cosine_scaled_reward": 0.0013177692890167236, | |
| "step": 82 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2935.8958435058594, | |
| "epoch": 0.09485714285714286, | |
| "grad_norm": 0.21173416078090668, | |
| "kl": 0.0016698837280273438, | |
| "learning_rate": 7.612622032536507e-07, | |
| "loss": 0.0217, | |
| "reward": -0.1123672490939498, | |
| "reward_std": 0.27851989958435297, | |
| "rewards/accuracy_reward": 0.0625, | |
| "rewards/cosine_scaled_reward": -0.17486725561320782, | |
| "step": 83 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3135.541702270508, | |
| "epoch": 0.096, | |
| "grad_norm": 0.16200174391269684, | |
| "kl": 0.0007222890853881836, | |
| "learning_rate": 7.54295724882796e-07, | |
| "loss": 0.0357, | |
| "reward": 0.19974466552957892, | |
| "reward_std": 0.48104337602853775, | |
| "rewards/accuracy_reward": 0.25000000186264515, | |
| "rewards/cosine_scaled_reward": -0.05025534983724356, | |
| "step": 84 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3218.375045776367, | |
| "epoch": 0.09714285714285714, | |
| "grad_norm": 0.15586286783218384, | |
| "kl": 0.0010872483253479004, | |
| "learning_rate": 7.472670160550848e-07, | |
| "loss": 0.0099, | |
| "reward": 0.09634784981608391, | |
| "reward_std": 0.6995302718132734, | |
| "rewards/accuracy_reward": 0.18750000558793545, | |
| "rewards/cosine_scaled_reward": -0.09115215530619025, | |
| "step": 85 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3214.8958587646484, | |
| "epoch": 0.09828571428571428, | |
| "grad_norm": 0.16115757822990417, | |
| "kl": 0.0023956298828125, | |
| "learning_rate": 7.401782177833147e-07, | |
| "loss": 0.0367, | |
| "reward": 0.044234735891222954, | |
| "reward_std": 0.4710828922688961, | |
| "rewards/accuracy_reward": 0.14583333767950535, | |
| "rewards/cosine_scaled_reward": -0.10159859777195379, | |
| "step": 86 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3030.041748046875, | |
| "epoch": 0.09942857142857142, | |
| "grad_norm": 0.23102359473705292, | |
| "kl": 0.007025090977549553, | |
| "learning_rate": 7.330314893841101e-07, | |
| "loss": 0.1063, | |
| "reward": 0.1451429482549429, | |
| "reward_std": 0.5002225376665592, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/cosine_scaled_reward": -0.0631904061883688, | |
| "step": 87 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3002.729248046875, | |
| "epoch": 0.10057142857142858, | |
| "grad_norm": 0.19206282496452332, | |
| "kl": 0.008913278579711914, | |
| "learning_rate": 7.258290078201731e-07, | |
| "loss": 0.0245, | |
| "reward": 0.16737306024879217, | |
| "reward_std": 0.46823734790086746, | |
| "rewards/accuracy_reward": 0.2083333358168602, | |
| "rewards/cosine_scaled_reward": -0.040960275335237384, | |
| "step": 88 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3434.5208435058594, | |
| "epoch": 0.10171428571428572, | |
| "grad_norm": 0.15944458544254303, | |
| "kl": 0.0026093721389770508, | |
| "learning_rate": 7.185729670371604e-07, | |
| "loss": 0.0369, | |
| "reward": 0.0876035988330841, | |
| "reward_std": 0.35983937978744507, | |
| "rewards/accuracy_reward": 0.1666666716337204, | |
| "rewards/cosine_scaled_reward": -0.07906308211386204, | |
| "step": 89 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2749.937515258789, | |
| "epoch": 0.10285714285714286, | |
| "grad_norm": 0.3216039538383484, | |
| "kl": 0.008787989616394043, | |
| "learning_rate": 7.11265577295385e-07, | |
| "loss": -0.007, | |
| "reward": -0.27455414831638336, | |
| "reward_std": 0.2550176875665784, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.29538749624043703, | |
| "step": 90 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3049.4166870117188, | |
| "epoch": 0.104, | |
| "grad_norm": 0.21130341291427612, | |
| "kl": 0.003309965133666992, | |
| "learning_rate": 7.039090644965509e-07, | |
| "loss": 0.0554, | |
| "reward": 0.2680956181138754, | |
| "reward_std": 0.5306598274037242, | |
| "rewards/accuracy_reward": 0.22916667349636555, | |
| "rewards/cosine_scaled_reward": 0.038928942289203405, | |
| "step": 91 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3023.8959045410156, | |
| "epoch": 0.10514285714285715, | |
| "grad_norm": 0.22417984902858734, | |
| "kl": 0.0150984525680542, | |
| "learning_rate": 6.965056695057204e-07, | |
| "loss": 0.0827, | |
| "reward": 0.06401701085269451, | |
| "reward_std": 0.4534417111426592, | |
| "rewards/accuracy_reward": 0.16666666977107525, | |
| "rewards/cosine_scaled_reward": -0.10264966636896133, | |
| "step": 92 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3390.0, | |
| "epoch": 0.10628571428571429, | |
| "grad_norm": 0.16805854439735413, | |
| "kl": 0.002878427505493164, | |
| "learning_rate": 6.890576474687263e-07, | |
| "loss": 0.0208, | |
| "reward": -0.25338721089065075, | |
| "reward_std": 0.18274102546274662, | |
| "rewards/accuracy_reward": 0.0, | |
| "rewards/cosine_scaled_reward": -0.25338720716536045, | |
| "step": 93 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3132.7291870117188, | |
| "epoch": 0.10742857142857143, | |
| "grad_norm": 0.23390114307403564, | |
| "kl": 0.004430115222930908, | |
| "learning_rate": 6.815672671252315e-07, | |
| "loss": 0.0391, | |
| "reward": 0.06390632130205631, | |
| "reward_std": 0.2537277704104781, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.10276034381240606, | |
| "step": 94 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3494.0208435058594, | |
| "epoch": 0.10857142857142857, | |
| "grad_norm": 0.13700152933597565, | |
| "kl": 0.0011189579963684082, | |
| "learning_rate": 6.740368101176495e-07, | |
| "loss": 0.0066, | |
| "reward": -0.24112762324512005, | |
| "reward_std": 0.25422961357980967, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.2619609571993351, | |
| "step": 95 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3054.812530517578, | |
| "epoch": 0.10971428571428571, | |
| "grad_norm": 0.1651657223701477, | |
| "kl": 0.00439077615737915, | |
| "learning_rate": 6.664685702961344e-07, | |
| "loss": 0.0172, | |
| "reward": 0.043599123833701015, | |
| "reward_std": 0.5089055569842458, | |
| "rewards/accuracy_reward": 0.16666667349636555, | |
| "rewards/cosine_scaled_reward": -0.12306755485769827, | |
| "step": 96 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3421.5833740234375, | |
| "epoch": 0.11085714285714286, | |
| "grad_norm": 0.1608337163925171, | |
| "kl": 0.00201570987701416, | |
| "learning_rate": 6.588648530198504e-07, | |
| "loss": 0.0159, | |
| "reward": 0.07680944725871086, | |
| "reward_std": 0.4438377758488059, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/cosine_scaled_reward": -0.13152388390153646, | |
| "step": 97 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3099.375045776367, | |
| "epoch": 0.112, | |
| "grad_norm": 0.20414039492607117, | |
| "kl": 0.0015641450881958008, | |
| "learning_rate": 6.512279744547392e-07, | |
| "loss": 0.0324, | |
| "reward": 0.009776812046766281, | |
| "reward_std": 0.18312601745128632, | |
| "rewards/accuracy_reward": 0.125, | |
| "rewards/cosine_scaled_reward": -0.11522318236529827, | |
| "step": 98 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2837.979217529297, | |
| "epoch": 0.11314285714285714, | |
| "grad_norm": 0.21932294964790344, | |
| "kl": 0.0025873184204101562, | |
| "learning_rate": 6.435602608679916e-07, | |
| "loss": -0.043, | |
| "reward": 0.07655952998902649, | |
| "reward_std": 0.24549889098852873, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.06927378475666046, | |
| "step": 99 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3029.0208587646484, | |
| "epoch": 0.11428571428571428, | |
| "grad_norm": 0.17687632143497467, | |
| "kl": 0.005702972412109375, | |
| "learning_rate": 6.358640479194451e-07, | |
| "loss": 0.0584, | |
| "reward": 0.3614894598722458, | |
| "reward_std": 0.4475015373900533, | |
| "rewards/accuracy_reward": 0.3125000074505806, | |
| "rewards/cosine_scaled_reward": 0.048989441245794296, | |
| "step": 100 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3111.666702270508, | |
| "epoch": 0.11542857142857142, | |
| "grad_norm": 0.19027626514434814, | |
| "kl": 0.0019845962524414062, | |
| "learning_rate": 6.281416799501187e-07, | |
| "loss": 0.025, | |
| "reward": 0.06912354379892349, | |
| "reward_std": 0.28665477968752384, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.09754312154836953, | |
| "step": 101 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2907.312530517578, | |
| "epoch": 0.11657142857142858, | |
| "grad_norm": 0.2261529117822647, | |
| "kl": 0.006736278533935547, | |
| "learning_rate": 6.203955092681039e-07, | |
| "loss": 0.0423, | |
| "reward": 0.11842460185289383, | |
| "reward_std": 0.49445752426981926, | |
| "rewards/accuracy_reward": 0.16666667349636555, | |
| "rewards/cosine_scaled_reward": -0.0482420613989234, | |
| "step": 102 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3219.6458587646484, | |
| "epoch": 0.11771428571428572, | |
| "grad_norm": 0.20831391215324402, | |
| "kl": 0.004827260971069336, | |
| "learning_rate": 6.126278954320294e-07, | |
| "loss": 0.065, | |
| "reward": -0.018325381679460406, | |
| "reward_std": 0.4717335421591997, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/cosine_scaled_reward": -0.14332537725567818, | |
| "step": 103 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2912.9791870117188, | |
| "epoch": 0.11885714285714286, | |
| "grad_norm": 0.17986515164375305, | |
| "kl": 0.034677982330322266, | |
| "learning_rate": 6.048412045323164e-07, | |
| "loss": 0.0421, | |
| "reward": -0.11950392462313175, | |
| "reward_std": 0.2809145264327526, | |
| "rewards/accuracy_reward": 0.0625, | |
| "rewards/cosine_scaled_reward": -0.18200392089784145, | |
| "step": 104 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3010.8958435058594, | |
| "epoch": 0.12, | |
| "grad_norm": 0.21148350834846497, | |
| "kl": 0.0034750699996948242, | |
| "learning_rate": 5.97037808470444e-07, | |
| "loss": 0.012, | |
| "reward": 0.14130431599915028, | |
| "reward_std": 0.47796807438135147, | |
| "rewards/accuracy_reward": 0.18750000558793545, | |
| "rewards/cosine_scaled_reward": -0.04619568458292633, | |
| "step": 105 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2606.0416870117188, | |
| "epoch": 0.12114285714285715, | |
| "grad_norm": 0.18517449498176575, | |
| "kl": 0.014556169509887695, | |
| "learning_rate": 5.892200842364462e-07, | |
| "loss": 0.0554, | |
| "reward": 0.554547032341361, | |
| "reward_std": 0.45337705919519067, | |
| "rewards/accuracy_reward": 0.41666666977107525, | |
| "rewards/cosine_scaled_reward": 0.1378803295083344, | |
| "step": 106 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3122.312515258789, | |
| "epoch": 0.12228571428571429, | |
| "grad_norm": 0.2288726419210434, | |
| "kl": 0.0027909278869628906, | |
| "learning_rate": 5.813904131848564e-07, | |
| "loss": 0.0421, | |
| "reward": 0.014997333288192749, | |
| "reward_std": 0.35810419358313084, | |
| "rewards/accuracy_reward": 0.1458333395421505, | |
| "rewards/cosine_scaled_reward": -0.1308360043913126, | |
| "step": 107 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3150.2083587646484, | |
| "epoch": 0.12342857142857143, | |
| "grad_norm": 0.22030171751976013, | |
| "kl": 0.0030778050422668457, | |
| "learning_rate": 5.735511803093248e-07, | |
| "loss": 0.0993, | |
| "reward": 0.12523731286637485, | |
| "reward_std": 0.47955003939568996, | |
| "rewards/accuracy_reward": 0.18750000558793545, | |
| "rewards/cosine_scaled_reward": -0.06226269342005253, | |
| "step": 108 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3190.062530517578, | |
| "epoch": 0.12457142857142857, | |
| "grad_norm": 0.16835717856884003, | |
| "kl": 0.001190185546875, | |
| "learning_rate": 5.657047735161255e-07, | |
| "loss": 0.0296, | |
| "reward": 0.11127902567386627, | |
| "reward_std": 0.30289303325116634, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.05538764409720898, | |
| "step": 109 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3129.8750762939453, | |
| "epoch": 0.12571428571428572, | |
| "grad_norm": 0.18525457382202148, | |
| "kl": 0.0027114152908325195, | |
| "learning_rate": 5.578535828967777e-07, | |
| "loss": 0.0241, | |
| "reward": 0.1006783633492887, | |
| "reward_std": 0.5584467053413391, | |
| "rewards/accuracy_reward": 0.18750000558793545, | |
| "rewards/cosine_scaled_reward": -0.08682164316996932, | |
| "step": 110 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3500.687530517578, | |
| "epoch": 0.12685714285714286, | |
| "grad_norm": 0.16640202701091766, | |
| "kl": 0.003269195556640625, | |
| "learning_rate": 5.5e-07, | |
| "loss": 0.0407, | |
| "reward": 0.06599505990743637, | |
| "reward_std": 0.5910971527919173, | |
| "rewards/accuracy_reward": 0.16666666977107525, | |
| "rewards/cosine_scaled_reward": -0.10067159991012886, | |
| "step": 111 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3325.8125610351562, | |
| "epoch": 0.128, | |
| "grad_norm": 0.16706104576587677, | |
| "kl": 0.0014832019805908203, | |
| "learning_rate": 5.421464171032224e-07, | |
| "loss": 0.0229, | |
| "reward": 0.19466983899474144, | |
| "reward_std": 0.5429200492799282, | |
| "rewards/accuracy_reward": 0.2291666753590107, | |
| "rewards/cosine_scaled_reward": -0.03449681680649519, | |
| "step": 112 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3154.7916870117188, | |
| "epoch": 0.12914285714285714, | |
| "grad_norm": 0.18158937990665436, | |
| "kl": 0.011591911315917969, | |
| "learning_rate": 5.342952264838747e-07, | |
| "loss": 0.0169, | |
| "reward": 0.09023613715544343, | |
| "reward_std": 0.4198597203940153, | |
| "rewards/accuracy_reward": 0.16666667349636555, | |
| "rewards/cosine_scaled_reward": -0.07643054611980915, | |
| "step": 113 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2874.5000381469727, | |
| "epoch": 0.13028571428571428, | |
| "grad_norm": 0.15838727355003357, | |
| "kl": 0.003212451934814453, | |
| "learning_rate": 5.264488196906752e-07, | |
| "loss": -0.0056, | |
| "reward": -0.14509500935673714, | |
| "reward_std": 0.27863842714577913, | |
| "rewards/accuracy_reward": 0.0833333358168602, | |
| "rewards/cosine_scaled_reward": -0.22842834889888763, | |
| "step": 114 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2977.4791870117188, | |
| "epoch": 0.13142857142857142, | |
| "grad_norm": 0.20102404057979584, | |
| "kl": 0.005918979644775391, | |
| "learning_rate": 5.186095868151436e-07, | |
| "loss": 0.0203, | |
| "reward": 0.08041055500507355, | |
| "reward_std": 0.34110954217612743, | |
| "rewards/accuracy_reward": 0.1875, | |
| "rewards/cosine_scaled_reward": -0.10708944126963615, | |
| "step": 115 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3477.7708740234375, | |
| "epoch": 0.13257142857142856, | |
| "grad_norm": 0.1574009358882904, | |
| "kl": 0.002073526382446289, | |
| "learning_rate": 5.107799157635538e-07, | |
| "loss": -0.0103, | |
| "reward": 0.10368268750607967, | |
| "reward_std": 0.2848002705723047, | |
| "rewards/accuracy_reward": 0.1875, | |
| "rewards/cosine_scaled_reward": -0.08381731063127518, | |
| "step": 116 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3352.8125, | |
| "epoch": 0.1337142857142857, | |
| "grad_norm": 0.2221246063709259, | |
| "kl": 0.003772258758544922, | |
| "learning_rate": 5.02962191529556e-07, | |
| "loss": 0.046, | |
| "reward": -0.11389691138174385, | |
| "reward_std": 0.4279285650700331, | |
| "rewards/accuracy_reward": 0.06250000186264515, | |
| "rewards/cosine_scaled_reward": -0.17639691312797368, | |
| "step": 117 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3320.604248046875, | |
| "epoch": 0.13485714285714287, | |
| "grad_norm": 0.19471098482608795, | |
| "kl": 0.002338886260986328, | |
| "learning_rate": 4.951587954676837e-07, | |
| "loss": 0.0529, | |
| "reward": 0.3609738126397133, | |
| "reward_std": 0.551423080265522, | |
| "rewards/accuracy_reward": 0.31250000558793545, | |
| "rewards/cosine_scaled_reward": 0.04847381100989878, | |
| "step": 118 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2834.875030517578, | |
| "epoch": 0.136, | |
| "grad_norm": 0.18936385214328766, | |
| "kl": 0.013357400894165039, | |
| "learning_rate": 4.873721045679706e-07, | |
| "loss": 0.0535, | |
| "reward": -0.012741273269057274, | |
| "reward_std": 0.41901483573019505, | |
| "rewards/accuracy_reward": 0.1250000037252903, | |
| "rewards/cosine_scaled_reward": -0.13774127699434757, | |
| "step": 119 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2937.7916984558105, | |
| "epoch": 0.13714285714285715, | |
| "grad_norm": 0.24001307785511017, | |
| "kl": 0.007732391357421875, | |
| "learning_rate": 4.79604490731896e-07, | |
| "loss": -0.0484, | |
| "reward": -0.006065825000405312, | |
| "reward_std": 0.15845976071432233, | |
| "rewards/accuracy_reward": 0.125, | |
| "rewards/cosine_scaled_reward": -0.13106583431363106, | |
| "step": 120 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2497.1042098999023, | |
| "epoch": 0.1382857142857143, | |
| "grad_norm": 0.17908261716365814, | |
| "kl": 0.026035308837890625, | |
| "learning_rate": 4.7185832004988133e-07, | |
| "loss": 0.0151, | |
| "reward": 0.11876576207578182, | |
| "reward_std": 0.3727921061217785, | |
| "rewards/accuracy_reward": 0.18750000186264515, | |
| "rewards/cosine_scaled_reward": -0.06873424537479877, | |
| "step": 121 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3167.729217529297, | |
| "epoch": 0.13942857142857143, | |
| "grad_norm": 0.20142368972301483, | |
| "kl": 0.0050547122955322266, | |
| "learning_rate": 4.641359520805548e-07, | |
| "loss": 0.0011, | |
| "reward": 0.2649596631526947, | |
| "reward_std": 0.5385044906288385, | |
| "rewards/accuracy_reward": 0.25000000931322575, | |
| "rewards/cosine_scaled_reward": 0.014959652442485094, | |
| "step": 122 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3255.562530517578, | |
| "epoch": 0.14057142857142857, | |
| "grad_norm": 0.15558990836143494, | |
| "kl": 0.0027800798416137695, | |
| "learning_rate": 4.5643973913200837e-07, | |
| "loss": 0.0281, | |
| "reward": -0.0696017425507307, | |
| "reward_std": 0.36961269890889525, | |
| "rewards/accuracy_reward": 0.1041666679084301, | |
| "rewards/cosine_scaled_reward": -0.17376840766519308, | |
| "step": 123 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2784.2709045410156, | |
| "epoch": 0.1417142857142857, | |
| "grad_norm": 0.2072015255689621, | |
| "kl": 0.026355981826782227, | |
| "learning_rate": 4.4877202554526084e-07, | |
| "loss": 0.0621, | |
| "reward": 0.23046518303453922, | |
| "reward_std": 0.6176506169140339, | |
| "rewards/accuracy_reward": 0.2500000074505806, | |
| "rewards/cosine_scaled_reward": -0.019534816965460777, | |
| "step": 124 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2942.375, | |
| "epoch": 0.14285714285714285, | |
| "grad_norm": 0.17982633411884308, | |
| "kl": 0.07889890670776367, | |
| "learning_rate": 4.4113514698014953e-07, | |
| "loss": -0.0356, | |
| "reward": 0.20839283242821693, | |
| "reward_std": 0.37462584115564823, | |
| "rewards/accuracy_reward": 0.22916666977107525, | |
| "rewards/cosine_scaled_reward": -0.02077384665608406, | |
| "step": 125 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2955.916702270508, | |
| "epoch": 0.144, | |
| "grad_norm": 0.16790178418159485, | |
| "kl": 0.0015211105346679688, | |
| "learning_rate": 4.3353142970386557e-07, | |
| "loss": 0.0345, | |
| "reward": 0.08628061786293983, | |
| "reward_std": 0.33532324619591236, | |
| "rewards/accuracy_reward": 0.2083333432674408, | |
| "rewards/cosine_scaled_reward": -0.12205271050333977, | |
| "step": 126 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3365.2083435058594, | |
| "epoch": 0.14514285714285713, | |
| "grad_norm": 0.24060513079166412, | |
| "kl": 0.0030236244201660156, | |
| "learning_rate": 4.2596318988235037e-07, | |
| "loss": 0.0356, | |
| "reward": -0.12239565665367991, | |
| "reward_std": 0.304832368157804, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.16406232165172696, | |
| "step": 127 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3064.354217529297, | |
| "epoch": 0.1462857142857143, | |
| "grad_norm": 0.17565199732780457, | |
| "kl": 0.004384756088256836, | |
| "learning_rate": 4.1843273287476854e-07, | |
| "loss": 0.0906, | |
| "reward": 0.28212980553507805, | |
| "reward_std": 0.6925715450197458, | |
| "rewards/accuracy_reward": 0.27083333767950535, | |
| "rewards/cosine_scaled_reward": 0.011296482291072607, | |
| "step": 128 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3541.0625, | |
| "epoch": 0.14742857142857144, | |
| "grad_norm": 0.15770180523395538, | |
| "kl": 0.0032634735107421875, | |
| "learning_rate": 4.1094235253127374e-07, | |
| "loss": 0.0054, | |
| "reward": -0.060815065167844296, | |
| "reward_std": 0.25299688428640366, | |
| "rewards/accuracy_reward": 0.1041666716337204, | |
| "rewards/cosine_scaled_reward": -0.1649817330762744, | |
| "step": 129 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3418.1458435058594, | |
| "epoch": 0.14857142857142858, | |
| "grad_norm": 0.16763123869895935, | |
| "kl": 0.0033943653106689453, | |
| "learning_rate": 4.034943304942796e-07, | |
| "loss": 0.0367, | |
| "reward": 0.07925572944805026, | |
| "reward_std": 0.4248347468674183, | |
| "rewards/accuracy_reward": 0.1458333395421505, | |
| "rewards/cosine_scaled_reward": -0.06657761428505182, | |
| "step": 130 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3062.2292098999023, | |
| "epoch": 0.14971428571428572, | |
| "grad_norm": 0.169542133808136, | |
| "kl": 0.019721031188964844, | |
| "learning_rate": 3.9609093550344907e-07, | |
| "loss": -0.0339, | |
| "reward": 0.32605051435530186, | |
| "reward_std": 0.5189148262143135, | |
| "rewards/accuracy_reward": 0.2916666716337204, | |
| "rewards/cosine_scaled_reward": 0.034383836667984724, | |
| "step": 131 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3312.1875610351562, | |
| "epoch": 0.15085714285714286, | |
| "grad_norm": 0.1781223863363266, | |
| "kl": 0.0033979415893554688, | |
| "learning_rate": 3.8873442270461485e-07, | |
| "loss": 0.0421, | |
| "reward": 0.27052065124735236, | |
| "reward_std": 0.7929235212504864, | |
| "rewards/accuracy_reward": 0.27083334140479565, | |
| "rewards/cosine_scaled_reward": -0.0003126785159111023, | |
| "step": 132 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3373.3541870117188, | |
| "epoch": 0.152, | |
| "grad_norm": 0.19108432531356812, | |
| "kl": 0.0029697418212890625, | |
| "learning_rate": 3.8142703296283953e-07, | |
| "loss": -0.0076, | |
| "reward": -0.12809189222753048, | |
| "reward_std": 0.2327743861824274, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.2739252429455519, | |
| "step": 133 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3061.6666870117188, | |
| "epoch": 0.15314285714285714, | |
| "grad_norm": 0.1890539675951004, | |
| "kl": 0.01691436767578125, | |
| "learning_rate": 3.7417099217982686e-07, | |
| "loss": -0.0643, | |
| "reward": -0.005162534303963184, | |
| "reward_std": 0.384098834823817, | |
| "rewards/accuracy_reward": 0.1458333395421505, | |
| "rewards/cosine_scaled_reward": -0.15099586872383952, | |
| "step": 134 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2299.3333892822266, | |
| "epoch": 0.15428571428571428, | |
| "grad_norm": 0.2244781255722046, | |
| "kl": 0.014550924301147461, | |
| "learning_rate": 3.6696851061588994e-07, | |
| "loss": 0.0863, | |
| "reward": 0.7467391490936279, | |
| "reward_std": 0.5308007836574689, | |
| "rewards/accuracy_reward": 0.5000000093132257, | |
| "rewards/cosine_scaled_reward": 0.24673915409948677, | |
| "step": 135 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3300.187530517578, | |
| "epoch": 0.15542857142857142, | |
| "grad_norm": 0.18033601343631744, | |
| "kl": 0.004876136779785156, | |
| "learning_rate": 3.5982178221668533e-07, | |
| "loss": 0.0506, | |
| "reward": 0.16450288891792297, | |
| "reward_std": 0.5824453700333834, | |
| "rewards/accuracy_reward": 0.20833333767950535, | |
| "rewards/cosine_scaled_reward": -0.04383046319708228, | |
| "step": 136 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3515.1041870117188, | |
| "epoch": 0.15657142857142858, | |
| "grad_norm": 0.14998017251491547, | |
| "kl": 0.005837440490722656, | |
| "learning_rate": 3.5273298394491515e-07, | |
| "loss": 0.0241, | |
| "reward": -0.27306424523703754, | |
| "reward_std": 0.2572753308340907, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.29389757476747036, | |
| "step": 137 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3021.7083740234375, | |
| "epoch": 0.15771428571428572, | |
| "grad_norm": 0.17564791440963745, | |
| "kl": 0.0214078426361084, | |
| "learning_rate": 3.45704275117204e-07, | |
| "loss": -0.0187, | |
| "reward": 0.028833418153226376, | |
| "reward_std": 0.3300330266356468, | |
| "rewards/accuracy_reward": 0.12500000558793545, | |
| "rewards/cosine_scaled_reward": -0.09616657719016075, | |
| "step": 138 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3443.187530517578, | |
| "epoch": 0.15885714285714286, | |
| "grad_norm": 0.17364783585071564, | |
| "kl": 0.003993988037109375, | |
| "learning_rate": 3.387377967463493e-07, | |
| "loss": -0.0154, | |
| "reward": -0.1509188860654831, | |
| "reward_std": 0.3400895958766341, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.19258555606938899, | |
| "step": 139 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3458.5833740234375, | |
| "epoch": 0.16, | |
| "grad_norm": 0.2179875522851944, | |
| "kl": 0.009866714477539062, | |
| "learning_rate": 3.3183567088914833e-07, | |
| "loss": -0.0423, | |
| "reward": 0.11468263063579798, | |
| "reward_std": 0.3303603548556566, | |
| "rewards/accuracy_reward": 0.1666666679084301, | |
| "rewards/cosine_scaled_reward": -0.05198403773829341, | |
| "step": 140 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3385.3958740234375, | |
| "epoch": 0.16114285714285714, | |
| "grad_norm": 0.17362990975379944, | |
| "kl": 0.0052642822265625, | |
| "learning_rate": 3.250000000000001e-07, | |
| "loss": -0.0233, | |
| "reward": 0.12504707090556622, | |
| "reward_std": 0.5917222537100315, | |
| "rewards/accuracy_reward": 0.20833333767950535, | |
| "rewards/cosine_scaled_reward": -0.08328627422451973, | |
| "step": 141 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3200.6459045410156, | |
| "epoch": 0.16228571428571428, | |
| "grad_norm": 0.19476763904094696, | |
| "kl": 0.005651950836181641, | |
| "learning_rate": 3.182328662904756e-07, | |
| "loss": 0.0829, | |
| "reward": -0.07443058118224144, | |
| "reward_std": 0.4768957942724228, | |
| "rewards/accuracy_reward": 0.1250000037252903, | |
| "rewards/cosine_scaled_reward": -0.1994305873522535, | |
| "step": 142 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3258.6875610351562, | |
| "epoch": 0.16342857142857142, | |
| "grad_norm": 0.23263806104660034, | |
| "kl": 0.00710296630859375, | |
| "learning_rate": 3.115363310950578e-07, | |
| "loss": 0.0686, | |
| "reward": -0.07686329772695899, | |
| "reward_std": 0.355038208886981, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/cosine_scaled_reward": -0.18102996051311493, | |
| "step": 143 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3194.75, | |
| "epoch": 0.16457142857142856, | |
| "grad_norm": 0.17562638223171234, | |
| "kl": 0.0062198638916015625, | |
| "learning_rate": 3.0491243424323783e-07, | |
| "loss": -0.0268, | |
| "reward": -0.06668785959482193, | |
| "reward_std": 0.3318067956715822, | |
| "rewards/accuracy_reward": 0.12500000558793545, | |
| "rewards/cosine_scaled_reward": -0.19168786704540253, | |
| "step": 144 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2559.2500076293945, | |
| "epoch": 0.1657142857142857, | |
| "grad_norm": 0.24671532213687897, | |
| "kl": 0.009662628173828125, | |
| "learning_rate": 2.9836319343816397e-07, | |
| "loss": -0.0379, | |
| "reward": 0.37313266890123487, | |
| "reward_std": 0.33964273426681757, | |
| "rewards/accuracy_reward": 0.31250000186264515, | |
| "rewards/cosine_scaled_reward": 0.060632668901234865, | |
| "step": 145 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3086.375015258789, | |
| "epoch": 0.16685714285714287, | |
| "grad_norm": 0.15435300767421722, | |
| "kl": 0.0024802684783935547, | |
| "learning_rate": 2.918906036420294e-07, | |
| "loss": 0.0125, | |
| "reward": -0.19754038006067276, | |
| "reward_std": 0.2727511157281697, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.23920704424381256, | |
| "step": 146 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3410.8541870117188, | |
| "epoch": 0.168, | |
| "grad_norm": 0.2101191282272339, | |
| "kl": 0.004631996154785156, | |
| "learning_rate": 2.854966364683872e-07, | |
| "loss": 0.0229, | |
| "reward": -0.13149442267604172, | |
| "reward_std": 0.3892161212861538, | |
| "rewards/accuracy_reward": 0.06250000186264515, | |
| "rewards/cosine_scaled_reward": -0.19399441499263048, | |
| "step": 147 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2876.1250610351562, | |
| "epoch": 0.16914285714285715, | |
| "grad_norm": 0.18126218020915985, | |
| "kl": 0.008637428283691406, | |
| "learning_rate": 2.791832395815782e-07, | |
| "loss": 0.0202, | |
| "reward": -0.013470180332660675, | |
| "reward_std": 0.2522149607539177, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.15930352546274662, | |
| "step": 148 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3289.3750610351562, | |
| "epoch": 0.1702857142857143, | |
| "grad_norm": 0.16630330681800842, | |
| "kl": 0.004746913909912109, | |
| "learning_rate": 2.729523361034538e-07, | |
| "loss": 0.0387, | |
| "reward": 0.05808070907369256, | |
| "reward_std": 0.4934985414147377, | |
| "rewards/accuracy_reward": 0.16666666977107525, | |
| "rewards/cosine_scaled_reward": -0.10858596302568913, | |
| "step": 149 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3029.5417098999023, | |
| "epoch": 0.17142857142857143, | |
| "grad_norm": 0.16553689539432526, | |
| "kl": 0.007030487060546875, | |
| "learning_rate": 2.6680582402757324e-07, | |
| "loss": 0.0283, | |
| "reward": 0.3249948853626847, | |
| "reward_std": 0.5702541470527649, | |
| "rewards/accuracy_reward": 0.29166667722165585, | |
| "rewards/cosine_scaled_reward": 0.03332819044589996, | |
| "step": 150 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3161.3750610351562, | |
| "epoch": 0.17257142857142857, | |
| "grad_norm": 0.18477413058280945, | |
| "kl": 0.005667924880981445, | |
| "learning_rate": 2.6074557564105724e-07, | |
| "loss": 0.0607, | |
| "reward": 0.26313829235732555, | |
| "reward_std": 0.5168293453752995, | |
| "rewards/accuracy_reward": 0.25000000186264515, | |
| "rewards/cosine_scaled_reward": 0.013138292357325554, | |
| "step": 151 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3020.750011444092, | |
| "epoch": 0.1737142857142857, | |
| "grad_norm": 0.25039243698120117, | |
| "kl": 0.0038559436798095703, | |
| "learning_rate": 2.547734369542718e-07, | |
| "loss": -0.0343, | |
| "reward": -0.17000860534608364, | |
| "reward_std": 0.290488900616765, | |
| "rewards/accuracy_reward": 0.0416666679084301, | |
| "rewards/cosine_scaled_reward": -0.21167527325451374, | |
| "step": 152 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3283.4583587646484, | |
| "epoch": 0.17485714285714285, | |
| "grad_norm": 0.2211620956659317, | |
| "kl": 0.025887489318847656, | |
| "learning_rate": 2.488912271385139e-07, | |
| "loss": 0.0627, | |
| "reward": -0.1608979816082865, | |
| "reward_std": 0.30768819246441126, | |
| "rewards/accuracy_reward": 0.06250000186264515, | |
| "rewards/cosine_scaled_reward": -0.22339799162000418, | |
| "step": 153 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3509.625, | |
| "epoch": 0.176, | |
| "grad_norm": 0.15299846231937408, | |
| "kl": 0.0022106170654296875, | |
| "learning_rate": 2.4310073797187573e-07, | |
| "loss": 0.0166, | |
| "reward": 0.2281611803919077, | |
| "reward_std": 0.6786817125976086, | |
| "rewards/accuracy_reward": 0.2291666716337204, | |
| "rewards/cosine_scaled_reward": -0.0010055112652480602, | |
| "step": 154 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2825.2708587646484, | |
| "epoch": 0.17714285714285713, | |
| "grad_norm": 0.22317463159561157, | |
| "kl": 0.022104263305664062, | |
| "learning_rate": 2.374037332934512e-07, | |
| "loss": -0.0181, | |
| "reward": 0.3611069116741419, | |
| "reward_std": 0.5537243597209454, | |
| "rewards/accuracy_reward": 0.3125000111758709, | |
| "rewards/cosine_scaled_reward": 0.04860687721520662, | |
| "step": 155 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3509.3333435058594, | |
| "epoch": 0.1782857142857143, | |
| "grad_norm": 0.20389185845851898, | |
| "kl": 0.004364371299743652, | |
| "learning_rate": 2.3180194846605364e-07, | |
| "loss": 0.0435, | |
| "reward": -0.07293254625983536, | |
| "reward_std": 0.3572534155100584, | |
| "rewards/accuracy_reward": 0.08333333395421505, | |
| "rewards/cosine_scaled_reward": -0.15626587718725204, | |
| "step": 156 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3416.7291870117188, | |
| "epoch": 0.17942857142857144, | |
| "grad_norm": 0.16730445623397827, | |
| "kl": 0.005460262298583984, | |
| "learning_rate": 2.2629708984760706e-07, | |
| "loss": 0.0057, | |
| "reward": -0.07636097725480795, | |
| "reward_std": 0.2699170485138893, | |
| "rewards/accuracy_reward": 0.0833333358168602, | |
| "rewards/cosine_scaled_reward": -0.15969432052224874, | |
| "step": 157 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3422.0625610351562, | |
| "epoch": 0.18057142857142858, | |
| "grad_norm": 0.14682160317897797, | |
| "kl": 0.0034723281860351562, | |
| "learning_rate": 2.2089083427137329e-07, | |
| "loss": 0.0179, | |
| "reward": 0.32586812041699886, | |
| "reward_std": 0.3167658653110266, | |
| "rewards/accuracy_reward": 0.25000000558793545, | |
| "rewards/cosine_scaled_reward": 0.07586812693625689, | |
| "step": 158 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3442.6041870117188, | |
| "epoch": 0.18171428571428572, | |
| "grad_norm": 0.15924254059791565, | |
| "kl": 0.014659881591796875, | |
| "learning_rate": 2.1558482853517253e-07, | |
| "loss": 0.0161, | |
| "reward": -0.2088061198592186, | |
| "reward_std": 0.26268018409609795, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.2296394552104175, | |
| "step": 159 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3276.9791870117188, | |
| "epoch": 0.18285714285714286, | |
| "grad_norm": 0.1880086064338684, | |
| "kl": 0.008440017700195312, | |
| "learning_rate": 2.1038068889975259e-07, | |
| "loss": 0.0752, | |
| "reward": 0.1678475480293855, | |
| "reward_std": 0.5140087231993675, | |
| "rewards/accuracy_reward": 0.2291666753590107, | |
| "rewards/cosine_scaled_reward": -0.06131910812109709, | |
| "step": 160 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3382.1875610351562, | |
| "epoch": 0.184, | |
| "grad_norm": 0.1801372915506363, | |
| "kl": 0.015121936798095703, | |
| "learning_rate": 2.0528000059645995e-07, | |
| "loss": 0.0314, | |
| "reward": 0.0975832249969244, | |
| "reward_std": 0.4436509357765317, | |
| "rewards/accuracy_reward": 0.16666667349636555, | |
| "rewards/cosine_scaled_reward": -0.06908343941904604, | |
| "step": 161 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3415.6875, | |
| "epoch": 0.18514285714285714, | |
| "grad_norm": 0.235793799161911, | |
| "kl": 0.00775146484375, | |
| "learning_rate": 2.0028431734436308e-07, | |
| "loss": 0.0693, | |
| "reward": 0.02907794527709484, | |
| "reward_std": 0.47673553973436356, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.11675538681447506, | |
| "step": 162 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2728.625045776367, | |
| "epoch": 0.18628571428571428, | |
| "grad_norm": 0.2469216287136078, | |
| "kl": 0.004032135009765625, | |
| "learning_rate": 1.9539516087697517e-07, | |
| "loss": 0.1104, | |
| "reward": 0.4551220340654254, | |
| "reward_std": 0.6010043127462268, | |
| "rewards/accuracy_reward": 0.33333333767950535, | |
| "rewards/cosine_scaled_reward": 0.12178870104253292, | |
| "step": 163 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2826.9166870117188, | |
| "epoch": 0.18742857142857142, | |
| "grad_norm": 0.2510416507720947, | |
| "kl": 0.010884761810302734, | |
| "learning_rate": 1.9061402047871833e-07, | |
| "loss": 0.0495, | |
| "reward": 0.29104864224791527, | |
| "reward_std": 0.41844242811203003, | |
| "rewards/accuracy_reward": 0.2708333395421505, | |
| "rewards/cosine_scaled_reward": 0.020215285941958427, | |
| "step": 164 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3338.875030517578, | |
| "epoch": 0.18857142857142858, | |
| "grad_norm": 0.17247295379638672, | |
| "kl": 0.005005836486816406, | |
| "learning_rate": 1.8594235253127372e-07, | |
| "loss": 0.0091, | |
| "reward": -0.10422082059085369, | |
| "reward_std": 0.4068410564213991, | |
| "rewards/accuracy_reward": 0.08333333395421505, | |
| "rewards/cosine_scaled_reward": -0.18755416944622993, | |
| "step": 165 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3450.0833740234375, | |
| "epoch": 0.18971428571428572, | |
| "grad_norm": 0.13382409512996674, | |
| "kl": 0.002165555953979492, | |
| "learning_rate": 1.8138158006995363e-07, | |
| "loss": 0.0106, | |
| "reward": 0.05564507842063904, | |
| "reward_std": 0.5360854268074036, | |
| "rewards/accuracy_reward": 0.1458333358168602, | |
| "rewards/cosine_scaled_reward": -0.09018826205283403, | |
| "step": 166 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2906.791748046875, | |
| "epoch": 0.19085714285714286, | |
| "grad_norm": 0.17852716147899628, | |
| "kl": 0.008731842041015625, | |
| "learning_rate": 1.7693309235023127e-07, | |
| "loss": 0.0562, | |
| "reward": 0.060366696037817746, | |
| "reward_std": 0.4651718642562628, | |
| "rewards/accuracy_reward": 0.14583333395421505, | |
| "rewards/cosine_scaled_reward": -0.08546665217727423, | |
| "step": 167 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3323.2291870117188, | |
| "epoch": 0.192, | |
| "grad_norm": 0.14015178382396698, | |
| "kl": 0.00559234619140625, | |
| "learning_rate": 1.7259824442455923e-07, | |
| "loss": 0.0001, | |
| "reward": -0.00011509843170642853, | |
| "reward_std": 0.48929045908153057, | |
| "rewards/accuracy_reward": 0.12500000186264515, | |
| "rewards/cosine_scaled_reward": -0.12511511333286762, | |
| "step": 168 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2785.3333587646484, | |
| "epoch": 0.19314285714285714, | |
| "grad_norm": 0.17836594581604004, | |
| "kl": 0.0029582977294921875, | |
| "learning_rate": 1.6837835672960831e-07, | |
| "loss": 0.0415, | |
| "reward": 0.7883107475936413, | |
| "reward_std": 0.31396112497895956, | |
| "rewards/accuracy_reward": 0.5416666679084301, | |
| "rewards/cosine_scaled_reward": 0.24664408713579178, | |
| "step": 169 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2772.9375228881836, | |
| "epoch": 0.19428571428571428, | |
| "grad_norm": 0.1806839257478714, | |
| "kl": 0.006674766540527344, | |
| "learning_rate": 1.6427471468404952e-07, | |
| "loss": 0.0447, | |
| "reward": 0.19183126650750637, | |
| "reward_std": 0.3847146909683943, | |
| "rewards/accuracy_reward": 0.2291666679084301, | |
| "rewards/cosine_scaled_reward": -0.03733538277447224, | |
| "step": 170 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3159.541717529297, | |
| "epoch": 0.19542857142857142, | |
| "grad_norm": 0.17659954726696014, | |
| "kl": 0.002955913543701172, | |
| "learning_rate": 1.6028856829700258e-07, | |
| "loss": 0.0338, | |
| "reward": 0.261606702581048, | |
| "reward_std": 0.4677132572978735, | |
| "rewards/accuracy_reward": 0.2500000074505806, | |
| "rewards/cosine_scaled_reward": 0.011606710031628609, | |
| "step": 171 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3158.8958435058594, | |
| "epoch": 0.19657142857142856, | |
| "grad_norm": 0.1956152766942978, | |
| "kl": 0.006173133850097656, | |
| "learning_rate": 1.5642113178727193e-07, | |
| "loss": 0.0097, | |
| "reward": 0.2144802352413535, | |
| "reward_std": 0.46392463706433773, | |
| "rewards/accuracy_reward": 0.2500000037252903, | |
| "rewards/cosine_scaled_reward": -0.035519770928658545, | |
| "step": 172 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2472.437530517578, | |
| "epoch": 0.1977142857142857, | |
| "grad_norm": 0.254905641078949, | |
| "kl": 0.008039474487304688, | |
| "learning_rate": 1.5267358321348285e-07, | |
| "loss": 0.0255, | |
| "reward": -0.041594887152314186, | |
| "reward_std": 0.4095227625221014, | |
| "rewards/accuracy_reward": 0.1250000037252903, | |
| "rewards/cosine_scaled_reward": -0.16659490356687456, | |
| "step": 173 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3101.9584045410156, | |
| "epoch": 0.19885714285714284, | |
| "grad_norm": 0.18561899662017822, | |
| "kl": 0.0075778961181640625, | |
| "learning_rate": 1.4904706411523448e-07, | |
| "loss": 0.035, | |
| "reward": 0.24006008356809616, | |
| "reward_std": 0.5043942555785179, | |
| "rewards/accuracy_reward": 0.22916666977107525, | |
| "rewards/cosine_scaled_reward": 0.010893410071730614, | |
| "step": 174 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2890.500030517578, | |
| "epoch": 0.2, | |
| "grad_norm": 0.17110423743724823, | |
| "kl": 0.01293182373046875, | |
| "learning_rate": 1.4554267916537495e-07, | |
| "loss": 0.0116, | |
| "reward": 0.32882001250982285, | |
| "reward_std": 0.2852998124435544, | |
| "rewards/accuracy_reward": 0.3125, | |
| "rewards/cosine_scaled_reward": 0.016320008784532547, | |
| "step": 175 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3115.0208892822266, | |
| "epoch": 0.20114285714285715, | |
| "grad_norm": 0.2752262055873871, | |
| "kl": 0.008016586303710938, | |
| "learning_rate": 1.4216149583350755e-07, | |
| "loss": 0.0955, | |
| "reward": 0.15877557545900345, | |
| "reward_std": 0.6403312431648374, | |
| "rewards/accuracy_reward": 0.20833333767950535, | |
| "rewards/cosine_scaled_reward": -0.049557752907276154, | |
| "step": 176 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3386.5833435058594, | |
| "epoch": 0.2022857142857143, | |
| "grad_norm": 0.1783076673746109, | |
| "kl": 0.019231557846069336, | |
| "learning_rate": 1.3890454406082956e-07, | |
| "loss": -0.0247, | |
| "reward": 0.05885206814855337, | |
| "reward_std": 0.3794629080221057, | |
| "rewards/accuracy_reward": 0.125, | |
| "rewards/cosine_scaled_reward": -0.06614794302731752, | |
| "step": 177 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3214.7291870117188, | |
| "epoch": 0.20342857142857143, | |
| "grad_norm": 0.1741504669189453, | |
| "kl": 0.021752357482910156, | |
| "learning_rate": 1.3577281594640182e-07, | |
| "loss": -0.019, | |
| "reward": 0.19468790292739868, | |
| "reward_std": 0.5135045610368252, | |
| "rewards/accuracy_reward": 0.2083333395421505, | |
| "rewards/cosine_scaled_reward": -0.013645432889461517, | |
| "step": 178 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3233.4166870117188, | |
| "epoch": 0.20457142857142857, | |
| "grad_norm": 0.16726212203502655, | |
| "kl": 0.0071485042572021484, | |
| "learning_rate": 1.3276726544494571e-07, | |
| "loss": 0.0061, | |
| "reward": -0.21683532558381557, | |
| "reward_std": 0.26457902044057846, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.23766866483492777, | |
| "step": 179 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2661.5000228881836, | |
| "epoch": 0.2057142857142857, | |
| "grad_norm": 0.30925920605659485, | |
| "kl": 0.015074729919433594, | |
| "learning_rate": 1.2988880807625927e-07, | |
| "loss": 0.0328, | |
| "reward": 0.512437904253602, | |
| "reward_std": 0.387176813557744, | |
| "rewards/accuracy_reward": 0.3958333358168602, | |
| "rewards/cosine_scaled_reward": 0.11660455353558064, | |
| "step": 180 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3386.437530517578, | |
| "epoch": 0.20685714285714285, | |
| "grad_norm": 0.2104531228542328, | |
| "kl": 0.0052337646484375, | |
| "learning_rate": 1.2713832064634125e-07, | |
| "loss": 0.0401, | |
| "reward": -0.050248525105416775, | |
| "reward_std": 0.3244944280013442, | |
| "rewards/accuracy_reward": 0.10416666977107525, | |
| "rewards/cosine_scaled_reward": -0.15441520046442747, | |
| "step": 181 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2823.104179382324, | |
| "epoch": 0.208, | |
| "grad_norm": 0.1794274002313614, | |
| "kl": 0.02399587631225586, | |
| "learning_rate": 1.2451664098030743e-07, | |
| "loss": -0.0603, | |
| "reward": 0.1379284020513296, | |
| "reward_std": 0.4621511232107878, | |
| "rewards/accuracy_reward": 0.18750000558793545, | |
| "rewards/cosine_scaled_reward": -0.049571617506444454, | |
| "step": 182 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3123.6458435058594, | |
| "epoch": 0.20914285714285713, | |
| "grad_norm": 0.17760169506072998, | |
| "kl": 0.023395538330078125, | |
| "learning_rate": 1.220245676671809e-07, | |
| "loss": 0.0078, | |
| "reward": 0.16959657333791256, | |
| "reward_std": 0.3798244297504425, | |
| "rewards/accuracy_reward": 0.2291666679084301, | |
| "rewards/cosine_scaled_reward": -0.059570083394646645, | |
| "step": 183 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3245.2708435058594, | |
| "epoch": 0.2102857142857143, | |
| "grad_norm": 0.19478560984134674, | |
| "kl": 0.005706787109375, | |
| "learning_rate": 1.1966285981663407e-07, | |
| "loss": -0.0293, | |
| "reward": -0.055914007127285004, | |
| "reward_std": 0.2492837980389595, | |
| "rewards/accuracy_reward": 0.1041666716337204, | |
| "rewards/cosine_scaled_reward": -0.1600806824862957, | |
| "step": 184 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3321.2708740234375, | |
| "epoch": 0.21142857142857144, | |
| "grad_norm": 0.17917311191558838, | |
| "kl": 0.008937358856201172, | |
| "learning_rate": 1.1743223682775649e-07, | |
| "loss": 0.0092, | |
| "reward": -0.04576564393937588, | |
| "reward_std": 0.39649237506091595, | |
| "rewards/accuracy_reward": 0.1041666679084301, | |
| "rewards/cosine_scaled_reward": -0.14993230905383825, | |
| "step": 185 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3297.6458740234375, | |
| "epoch": 0.21257142857142858, | |
| "grad_norm": 0.1722106635570526, | |
| "kl": 0.003765106201171875, | |
| "learning_rate": 1.1533337816991931e-07, | |
| "loss": 0.0326, | |
| "reward": 0.4093305990099907, | |
| "reward_std": 0.3512581354007125, | |
| "rewards/accuracy_reward": 0.3333333432674408, | |
| "rewards/cosine_scaled_reward": 0.0759972594678402, | |
| "step": 186 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3079.541717529297, | |
| "epoch": 0.21371428571428572, | |
| "grad_norm": 0.27707189321517944, | |
| "kl": 0.00933837890625, | |
| "learning_rate": 1.1336692317580158e-07, | |
| "loss": -0.0215, | |
| "reward": -0.08524070866405964, | |
| "reward_std": 0.38815517351031303, | |
| "rewards/accuracy_reward": 0.08333333395421505, | |
| "rewards/cosine_scaled_reward": -0.16857404820621014, | |
| "step": 187 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3571.2916870117188, | |
| "epoch": 0.21485714285714286, | |
| "grad_norm": 0.1520886868238449, | |
| "kl": 0.0035920143127441406, | |
| "learning_rate": 1.1153347084664419e-07, | |
| "loss": 0.002, | |
| "reward": -0.1857756022363901, | |
| "reward_std": 0.2816652413457632, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.20660893432796001, | |
| "step": 188 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3236.5208587646484, | |
| "epoch": 0.216, | |
| "grad_norm": 0.19879254698753357, | |
| "kl": 0.006503105163574219, | |
| "learning_rate": 1.0983357966978745e-07, | |
| "loss": 0.0444, | |
| "reward": -0.016374904662370682, | |
| "reward_std": 0.4003318352624774, | |
| "rewards/accuracy_reward": 0.1458333358168602, | |
| "rewards/cosine_scaled_reward": -0.1622082181274891, | |
| "step": 189 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3082.4583740234375, | |
| "epoch": 0.21714285714285714, | |
| "grad_norm": 0.16634653508663177, | |
| "kl": 0.00673985481262207, | |
| "learning_rate": 1.0826776744855121e-07, | |
| "loss": 0.0302, | |
| "reward": 0.24105202872306108, | |
| "reward_std": 0.5610934533178806, | |
| "rewards/accuracy_reward": 0.2708333358168602, | |
| "rewards/cosine_scaled_reward": -0.02978132525458932, | |
| "step": 190 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2954.7708740234375, | |
| "epoch": 0.21828571428571428, | |
| "grad_norm": 0.2336689531803131, | |
| "kl": 0.009424209594726562, | |
| "learning_rate": 1.068365111445064e-07, | |
| "loss": 0.121, | |
| "reward": 0.170956801623106, | |
| "reward_std": 0.3355375565588474, | |
| "rewards/accuracy_reward": 0.2083333432674408, | |
| "rewards/cosine_scaled_reward": -0.037376536056399345, | |
| "step": 191 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3462.2916870117188, | |
| "epoch": 0.21942857142857142, | |
| "grad_norm": 0.1604224145412445, | |
| "kl": 0.0033931732177734375, | |
| "learning_rate": 1.0554024673218806e-07, | |
| "loss": 0.0184, | |
| "reward": -0.08346282877027988, | |
| "reward_std": 0.28180871857330203, | |
| "rewards/accuracy_reward": 0.0833333358168602, | |
| "rewards/cosine_scaled_reward": -0.16679617203772068, | |
| "step": 192 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3324.479202270508, | |
| "epoch": 0.22057142857142858, | |
| "grad_norm": 0.23563383519649506, | |
| "kl": 0.004616260528564453, | |
| "learning_rate": 1.0437936906629334e-07, | |
| "loss": 0.0425, | |
| "reward": -0.1967187790432945, | |
| "reward_std": 0.2698577819392085, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.21755211427807808, | |
| "step": 193 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3275.7916870117188, | |
| "epoch": 0.22171428571428572, | |
| "grad_norm": 0.14834217727184296, | |
| "kl": 0.002389192581176758, | |
| "learning_rate": 1.0335423176140511e-07, | |
| "loss": -0.012, | |
| "reward": 0.502600722014904, | |
| "reward_std": 0.4794109221547842, | |
| "rewards/accuracy_reward": 0.3541666679084301, | |
| "rewards/cosine_scaled_reward": 0.14843403454869986, | |
| "step": 194 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3356.8541870117188, | |
| "epoch": 0.22285714285714286, | |
| "grad_norm": 0.16246356070041656, | |
| "kl": 0.0031523704528808594, | |
| "learning_rate": 1.0246514708427701e-07, | |
| "loss": 0.0019, | |
| "reward": 0.14120314689353108, | |
| "reward_std": 0.36129687167704105, | |
| "rewards/accuracy_reward": 0.20833333395421505, | |
| "rewards/cosine_scaled_reward": -0.06713018752634525, | |
| "step": 195 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3576.2291870117188, | |
| "epoch": 0.224, | |
| "grad_norm": 0.15277273952960968, | |
| "kl": 0.0033049583435058594, | |
| "learning_rate": 1.017123858587145e-07, | |
| "loss": 0.0026, | |
| "reward": -0.04466534685343504, | |
| "reward_std": 0.37323744408786297, | |
| "rewards/accuracy_reward": 0.08333333395421505, | |
| "rewards/cosine_scaled_reward": -0.12799868267029524, | |
| "step": 196 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2975.020835876465, | |
| "epoch": 0.22514285714285714, | |
| "grad_norm": 0.24708448350429535, | |
| "kl": 0.008154869079589844, | |
| "learning_rate": 1.0109617738307911e-07, | |
| "loss": 0.0281, | |
| "reward": 0.3771647736430168, | |
| "reward_std": 0.6537368502467871, | |
| "rewards/accuracy_reward": 0.2916666753590107, | |
| "rewards/cosine_scaled_reward": 0.08549809700343758, | |
| "step": 197 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3131.687530517578, | |
| "epoch": 0.22628571428571428, | |
| "grad_norm": 0.17502129077911377, | |
| "kl": 0.016117095947265625, | |
| "learning_rate": 1.0061670936044178e-07, | |
| "loss": -0.0111, | |
| "reward": 0.12027511559426785, | |
| "reward_std": 0.26299865636974573, | |
| "rewards/accuracy_reward": 0.2083333358168602, | |
| "rewards/cosine_scaled_reward": -0.08805821277201176, | |
| "step": 198 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 3566.75, | |
| "epoch": 0.22742857142857142, | |
| "grad_norm": 0.14533254504203796, | |
| "kl": 0.005496978759765625, | |
| "learning_rate": 1.002741278414069e-07, | |
| "loss": 0.0099, | |
| "reward": -0.23100331239402294, | |
| "reward_std": 0.262982377782464, | |
| "rewards/accuracy_reward": 0.02083333395421505, | |
| "rewards/cosine_scaled_reward": -0.25183664448559284, | |
| "step": 199 | |
| }, | |
| { | |
| "clip_fraction": 0.0, | |
| "completion_length": 2976.791717529297, | |
| "epoch": 0.22857142857142856, | |
| "grad_norm": 0.18751339614391327, | |
| "kl": 0.0031397342681884766, | |
| "learning_rate": 1.0006853717962393e-07, | |
| "loss": 0.0796, | |
| "reward": 0.2756543140858412, | |
| "reward_std": 0.398221081122756, | |
| "rewards/accuracy_reward": 0.2708333358168602, | |
| "rewards/cosine_scaled_reward": 0.004820965230464935, | |
| "step": 200 | |
| }, | |
| { | |
| "epoch": 0.22857142857142856, | |
| "step": 200, | |
| "total_flos": 0.0, | |
| "train_loss": 0.024151897984629613, | |
| "train_runtime": 32217.4606, | |
| "train_samples_per_second": 0.298, | |
| "train_steps_per_second": 0.006 | |
| } | |
| ], | |
| "logging_steps": 1, | |
| "max_steps": 200, | |
| "num_input_tokens_seen": 0, | |
| "num_train_epochs": 1, | |
| "save_steps": 10, | |
| "stateful_callbacks": { | |
| "TrainerControl": { | |
| "args": { | |
| "should_epoch_stop": false, | |
| "should_evaluate": false, | |
| "should_log": false, | |
| "should_save": true, | |
| "should_training_stop": true | |
| }, | |
| "attributes": {} | |
| } | |
| }, | |
| "total_flos": 0.0, | |
| "train_batch_size": 6, | |
| "trial_name": null, | |
| "trial_params": null | |
| } | |