{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.9558658800687029, "eval_steps": 500, "global_step": 3200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 229.75, "completions/mean_length": 1020.9397430419922, "completions/mean_terminated_length": 170.3125, "completions/min_length": 854.25, "completions/min_terminated_length": 86.25, "epoch": 0.00029870808752146963, "grad_norm": 0.03279353305697441, "kl": 0.0, "learning_rate": 0.0, "loss": 0.0068, "num_tokens": 529493.0, "reward": 0.2566964328289032, "reward_std": 0.015117173083126545, "rewards/accuracy_reward/mean": 0.004464285913854837, "rewards/accuracy_reward/std": 0.033256832510232925, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2522321417927742, "rewards/tag_count_reward/std": 0.011650682426989079, "step": 1 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9732142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 710.75, "completions/mean_length": 1020.4777069091797, "completions/mean_terminated_length": 662.0916748046875, "completions/min_length": 821.0, "completions/min_terminated_length": 565.0, "epoch": 0.0005974161750429393, "grad_norm": 0.05416473001241684, "kl": 0.0, "learning_rate": 6.250000000000001e-08, "loss": 0.0049, "num_tokens": 1055211.0, "reward": 0.271205373108387, "reward_std": 0.0374436741694808, "rewards/accuracy_reward/mean": 0.006696428870782256, "rewards/accuracy_reward/std": 0.05687961168587208, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2645089253783226, "rewards/tag_count_reward/std": 0.04823805205523968, "step": 2 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 243.75, "completions/mean_length": 1023.5714416503906, "completions/mean_terminated_length": 240.0, "completions/min_length": 1000.5, "completions/min_terminated_length": 232.5, "epoch": 0.0008961242625644089, "grad_norm": 0.03999744728207588, "kl": 0.00026869773864746094, "learning_rate": 1.2500000000000002e-07, "loss": 0.0008, "num_tokens": 1590987.0, "reward": 0.2578125074505806, "reward_std": 0.01996421627700329, "rewards/accuracy_reward/mean": 0.0022321429569274187, "rewards/accuracy_reward/std": 0.023622779175639153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2555803582072258, "rewards/tag_count_reward/std": 0.02578705921769142, "step": 3 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.0011948323500858785, "grad_norm": 0.030198199674487114, "kl": 0.0002651214599609375, "learning_rate": 1.875e-07, "loss": 0.0, "num_tokens": 2129547.0, "reward": 0.2516741156578064, "reward_std": 0.005281830672174692, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2516741082072258, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 4 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 248.5, "completions/mean_length": 1011.6205444335938, "completions/mean_terminated_length": 183.0263214111328, "completions/min_length": 860.75, "completions/min_terminated_length": 92.75, "epoch": 0.0014935404376073482, "grad_norm": 0.0543970987200737, "kl": 0.00028228759765625, "learning_rate": 2.5000000000000004e-07, "loss": -0.0028, "num_tokens": 2653217.0, "reward": 0.2762276902794838, "reward_std": 0.042907388182356954, "rewards/accuracy_reward/mean": 0.008928571594879031, "rewards/accuracy_reward/std": 0.06416834704577923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2672991082072258, "rewards/tag_count_reward/std": 0.046106910333037376, "step": 5 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 455.25, "completions/mean_length": 1021.9129638671875, "completions/mean_terminated_length": 435.5, "completions/min_length": 930.25, "completions/min_terminated_length": 418.25, "epoch": 0.0017922485251288178, "grad_norm": 0.0554790273308754, "kl": 0.0002989768981933594, "learning_rate": 3.125e-07, "loss": 0.0032, "num_tokens": 3187194.0, "reward": 0.2912946566939354, "reward_std": 0.08426764979958534, "rewards/accuracy_reward/mean": 0.031622024020180106, "rewards/accuracy_reward/std": 0.14284836500883102, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2600446417927742, "rewards/tag_count_reward/std": 0.047143861185759306, "step": 6 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9977678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 247.5, "completions/mean_length": 1023.9241180419922, "completions/mean_terminated_length": 247.5, "completions/min_length": 1015.5, "completions/min_terminated_length": 247.5, "epoch": 0.0020909566126502874, "grad_norm": 0.03687676042318344, "kl": 0.0002837181091308594, "learning_rate": 3.75e-07, "loss": 0.0001, "num_tokens": 3726936.0, "reward": 0.2600446566939354, "reward_std": 0.019728224724531174, "rewards/accuracy_reward/mean": 0.0022321429569274187, "rewards/accuracy_reward/std": 0.023622779175639153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2578125, "rewards/tag_count_reward/std": 0.03462307108566165, "step": 7 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.002389664700171757, "grad_norm": 0.030483419075608253, "kl": 0.0002894401550292969, "learning_rate": 4.375e-07, "loss": 0.0, "num_tokens": 4254888.0, "reward": 0.2527901902794838, "reward_std": 0.0062251214403659105, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2527901753783226, "rewards/tag_count_reward/std": 0.017556377220898867, "step": 8 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9977678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 240.5, "completions/mean_length": 1023.8616180419922, "completions/mean_terminated_length": 240.5, "completions/min_length": 1008.5, "completions/min_terminated_length": 240.5, "epoch": 0.0026883727876932267, "grad_norm": 0.03618863224983215, "kl": 0.0002994537353515625, "learning_rate": 5.000000000000001e-07, "loss": 0.0002, "num_tokens": 4787482.0, "reward": 0.2606026828289032, "reward_std": 0.03040213417261839, "rewards/accuracy_reward/mean": 0.006696428870782256, "rewards/accuracy_reward/std": 0.05687961168587208, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.25390625, "rewards/tag_count_reward/std": 0.021279623731970787, "step": 9 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9821428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 452.75, "completions/mean_length": 1018.9776916503906, "completions/mean_terminated_length": 404.0833435058594, "completions/min_length": 871.25, "completions/min_terminated_length": 359.25, "epoch": 0.0029870808752146963, "grad_norm": 0.039917174726724625, "kl": 0.0002734661102294922, "learning_rate": 5.625e-07, "loss": 0.0037, "num_tokens": 5312128.0, "reward": 0.270647332072258, "reward_std": 0.03478178894147277, "rewards/accuracy_reward/mean": 0.011160714784637094, "rewards/accuracy_reward/std": 0.07022551260888577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2594866007566452, "rewards/tag_count_reward/std": 0.038242805283516645, "step": 10 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.96875, "completions/max_length": 1024.0, "completions/max_terminated_length": 712.25, "completions/mean_length": 1017.3861999511719, "completions/mean_terminated_length": 630.7361145019531, "completions/min_length": 790.5, "completions/min_terminated_length": 534.5, "epoch": 0.003285788962736166, "grad_norm": 0.043500471860170364, "kl": 0.0002751350402832031, "learning_rate": 6.25e-07, "loss": 0.0053, "num_tokens": 5838381.0, "reward": 0.2695312574505806, "reward_std": 0.01781349489465356, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.26953125, "rewards/tag_count_reward/std": 0.05734901875257492, "step": 11 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 238.75, "completions/mean_length": 1023.5133972167969, "completions/mean_terminated_length": 228.75, "completions/min_length": 986.75, "completions/min_terminated_length": 218.75, "epoch": 0.0035844970502576356, "grad_norm": 0.03667721524834633, "kl": 0.000255584716796875, "learning_rate": 6.875000000000001e-07, "loss": 0.0008, "num_tokens": 6363171.0, "reward": 0.2533482238650322, "reward_std": 0.00845726439729333, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2533482164144516, "rewards/tag_count_reward/std": 0.019964891485869884, "step": 12 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 475.25, "completions/mean_length": 1012.8951110839844, "completions/mean_terminated_length": 386.30357360839844, "completions/min_length": 825.75, "completions/min_terminated_length": 313.75, "epoch": 0.003883205137779105, "grad_norm": 0.04080239310860634, "kl": 0.0002799034118652344, "learning_rate": 7.5e-07, "loss": 0.0042, "num_tokens": 6886148.0, "reward": 0.2840401902794838, "reward_std": 0.02455357206054032, "rewards/accuracy_reward/mean": 0.02008928544819355, "rewards/accuracy_reward/std": 0.06826677173376083, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2639508917927742, "rewards/tag_count_reward/std": 0.03928397223353386, "step": 13 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.004181913225300575, "grad_norm": 0.03062329813838005, "kl": 0.0002834796905517578, "learning_rate": 8.125000000000001e-07, "loss": 0.0, "num_tokens": 7415524.0, "reward": 0.2522321492433548, "reward_std": 0.005831365240737796, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2522321417927742, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 14 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9665178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 712.25, "completions/mean_length": 1017.9263763427734, "completions/mean_terminated_length": 662.9791717529297, "completions/min_length": 836.5, "completions/min_terminated_length": 580.5, "epoch": 0.004480621312822045, "grad_norm": 0.0570688359439373, "kl": 0.00029087066650390625, "learning_rate": 8.75e-07, "loss": 0.0064, "num_tokens": 7941891.0, "reward": 0.3571428656578064, "reward_std": 0.10822875052690506, "rewards/accuracy_reward/mean": 0.0803571417927742, "rewards/accuracy_reward/std": 0.22719237208366394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2767857164144516, "rewards/tag_count_reward/std": 0.06333799287676811, "step": 15 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.004779329400343514, "grad_norm": 0.019299568608403206, "kl": 0.0002751350402832031, "learning_rate": 9.375000000000001e-07, "loss": 0.0, "num_tokens": 8473603.0, "reward": 0.2511160746216774, "reward_std": 0.0030496877152472734, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2511160746216774, "rewards/tag_count_reward/std": 0.008314208127558231, "step": 16 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.984375, "completions/max_length": 1024.0, "completions/max_terminated_length": 252.75, "completions/mean_length": 1022.4196472167969, "completions/mean_terminated_length": 230.71429443359375, "completions/min_length": 960.25, "completions/min_terminated_length": 192.25, "epoch": 0.005078037487864984, "grad_norm": 0.04471297562122345, "kl": 0.0002560615539550781, "learning_rate": 1.0000000000000002e-06, "loss": 0.0016, "num_tokens": 9006111.0, "reward": 0.2717634066939354, "reward_std": 0.039374695625156164, "rewards/accuracy_reward/mean": 0.013392857508733869, "rewards/accuracy_reward/std": 0.07548443600535393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2583705335855484, "rewards/tag_count_reward/std": 0.03744817804545164, "step": 17 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9709821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 249.25, "completions/mean_length": 1016.8459930419922, "completions/mean_terminated_length": 194.36538696289062, "completions/min_length": 919.25, "completions/min_terminated_length": 151.25, "epoch": 0.005376745575386453, "grad_norm": 0.045199159532785416, "kl": 0.0003008842468261719, "learning_rate": 1.0625e-06, "loss": 0.0022, "num_tokens": 9532762.0, "reward": 0.2678571566939354, "reward_std": 0.028510030591860414, "rewards/accuracy_reward/mean": 0.006696428870782256, "rewards/accuracy_reward/std": 0.05687961168587208, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.261160708963871, "rewards/tag_count_reward/std": 0.03893801709637046, "step": 18 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9464285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 494.25, "completions/mean_length": 1006.7656555175781, "completions/mean_terminated_length": 407.2159118652344, "completions/min_length": 840.5, "completions/min_terminated_length": 328.5, "epoch": 0.0056754536629079234, "grad_norm": 0.05608905106782913, "kl": 0.0002770423889160156, "learning_rate": 1.125e-06, "loss": 0.0022, "num_tokens": 10054385.0, "reward": 0.3415178582072258, "reward_std": 0.06047785095870495, "rewards/accuracy_reward/mean": 0.06473214249126613, "rewards/accuracy_reward/std": 0.1687364336103201, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.276785708963871, "rewards/tag_count_reward/std": 0.059488432481884956, "step": 19 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 254.75, "completions/mean_length": 1023.1428680419922, "completions/mean_terminated_length": 232.0, "completions/min_length": 978.75, "completions/min_terminated_length": 210.75, "epoch": 0.005974161750429393, "grad_norm": 0.057584911584854126, "kl": 0.00029754638671875, "learning_rate": 1.1875e-06, "loss": 0.0008, "num_tokens": 10588977.0, "reward": 0.2756696566939354, "reward_std": 0.036446330370381474, "rewards/accuracy_reward/mean": 0.02008928661234677, "rewards/accuracy_reward/std": 0.08829699270427227, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2555803507566452, "rewards/tag_count_reward/std": 0.027979944832623005, "step": 20 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 253.75, "completions/mean_length": 1019.8258972167969, "completions/mean_terminated_length": 217.0416717529297, "completions/min_length": 948.75, "completions/min_terminated_length": 180.75, "epoch": 0.006272869837950863, "grad_norm": 0.030625363811850548, "kl": 0.00028228759765625, "learning_rate": 1.25e-06, "loss": 0.0012, "num_tokens": 11119459.0, "reward": 0.2600446492433548, "reward_std": 0.005831365240737796, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2600446417927742, "rewards/tag_count_reward/std": 0.031518030911684036, "step": 21 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 214.25, "completions/mean_length": 1023.0401916503906, "completions/mean_terminated_length": 202.25, "completions/min_length": 958.25, "completions/min_terminated_length": 190.25, "epoch": 0.006571577925472332, "grad_norm": 0.03175712376832962, "kl": 0.0002713203430175781, "learning_rate": 1.3125000000000001e-06, "loss": 0.0008, "num_tokens": 11650981.0, "reward": 0.277901791036129, "reward_std": 0.03403337299823761, "rewards/accuracy_reward/mean": 0.0223214291036129, "rewards/accuracy_reward/std": 0.07160932570695877, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2555803582072258, "rewards/tag_count_reward/std": 0.017902331426739693, "step": 22 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9821428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 486.0, "completions/mean_length": 1021.3839569091797, "completions/mean_terminated_length": 435.4666748046875, "completions/min_length": 878.0, "completions/min_terminated_length": 366.0, "epoch": 0.006870286012993802, "grad_norm": 0.054987940937280655, "kl": 0.00028514862060546875, "learning_rate": 1.3750000000000002e-06, "loss": 0.0037, "num_tokens": 12177185.0, "reward": 0.2739955484867096, "reward_std": 0.046143966261297464, "rewards/accuracy_reward/mean": 0.013392857741564512, "rewards/accuracy_reward/std": 0.07985956594347954, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.260602667927742, "rewards/tag_count_reward/std": 0.0441946079954505, "step": 23 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 909.0, "completions/mean_length": 1021.357177734375, "completions/mean_terminated_length": 860.6041717529297, "completions/min_length": 799.0, "completions/min_terminated_length": 799.0, "epoch": 0.007168994100515271, "grad_norm": 0.05775587260723114, "kl": 0.0002803802490234375, "learning_rate": 1.4375e-06, "loss": 0.0027, "num_tokens": 12705297.0, "reward": 0.329799123108387, "reward_std": 0.07129301642999053, "rewards/accuracy_reward/mean": 0.058035715483129025, "rewards/accuracy_reward/std": 0.19588328152894974, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2717633843421936, "rewards/tag_count_reward/std": 0.06919946428388357, "step": 24 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.007467702188036741, "grad_norm": 0.035282012075185776, "kl": 0.0003056526184082031, "learning_rate": 1.5e-06, "loss": 0.0, "num_tokens": 13239553.0, "reward": 0.2656250074505806, "reward_std": 0.034724831115454435, "rewards/accuracy_reward/mean": 0.01116071455180645, "rewards/accuracy_reward/std": 0.051861658692359924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2544642835855484, "rewards/tag_count_reward/std": 0.023101806640625, "step": 25 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 710.5, "completions/mean_length": 1019.185302734375, "completions/mean_terminated_length": 610.7708435058594, "completions/min_length": 746.5, "completions/min_terminated_length": 490.5, "epoch": 0.00776641027555821, "grad_norm": 0.047262392938137054, "kl": 0.00030231475830078125, "learning_rate": 1.5625e-06, "loss": 0.004, "num_tokens": 13764708.0, "reward": 0.3186383992433548, "reward_std": 0.07342607341706753, "rewards/accuracy_reward/mean": 0.04910714365541935, "rewards/accuracy_reward/std": 0.18470034003257751, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.26953125, "rewards/tag_count_reward/std": 0.05734902247786522, "step": 26 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9888392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 486.75, "completions/mean_length": 1022.8080596923828, "completions/mean_terminated_length": 476.5625, "completions/min_length": 974.75, "completions/min_terminated_length": 462.75, "epoch": 0.00806511836307968, "grad_norm": 0.048997458070516586, "kl": 0.00032711029052734375, "learning_rate": 1.6250000000000001e-06, "loss": 0.0009, "num_tokens": 14298254.0, "reward": 0.2862723246216774, "reward_std": 0.041092000901699066, "rewards/accuracy_reward/mean": 0.02455357206054032, "rewards/accuracy_reward/std": 0.09523211047053337, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.26171875, "rewards/tag_count_reward/std": 0.042928848415613174, "step": 27 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9866071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 498.0, "completions/mean_length": 1023.1875305175781, "completions/mean_terminated_length": 489.6000061035156, "completions/min_length": 982.75, "completions/min_terminated_length": 470.75, "epoch": 0.00836382645060115, "grad_norm": 0.03670288994908333, "kl": 0.00030517578125, "learning_rate": 1.6875000000000001e-06, "loss": 0.0005, "num_tokens": 14831282.0, "reward": 0.262276791036129, "reward_std": 0.012166732456535101, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2622767835855484, "rewards/tag_count_reward/std": 0.04475103225558996, "step": 28 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 750.5, "completions/mean_length": 975.1116485595703, "completions/mean_terminated_length": 437.9386291503906, "completions/min_length": 501.0, "completions/min_terminated_length": 245.0, "epoch": 0.008662534538122619, "grad_norm": 0.0767391249537468, "kl": 0.0003314018249511719, "learning_rate": 1.75e-06, "loss": 0.0391, "num_tokens": 15339316.0, "reward": 0.3398437574505806, "reward_std": 0.06490222550928593, "rewards/accuracy_reward/mean": 0.055803569965064526, "rewards/accuracy_reward/std": 0.15079689025878906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2840401753783226, "rewards/tag_count_reward/std": 0.06876692920923233, "step": 29 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 760.0, "completions/mean_length": 1013.7812805175781, "completions/mean_terminated_length": 666.7812652587891, "completions/min_length": 812.75, "completions/min_terminated_length": 556.75, "epoch": 0.00896124262564409, "grad_norm": 0.05654367059469223, "kl": 0.000331878662109375, "learning_rate": 1.8125e-06, "loss": 0.0057, "num_tokens": 15864642.0, "reward": 0.2812500074505806, "reward_std": 0.031471846625208855, "rewards/accuracy_reward/mean": 0.008928571827709675, "rewards/accuracy_reward/std": 0.04660273343324661, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2723214253783226, "rewards/tag_count_reward/std": 0.05605909042060375, "step": 30 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 471.0, "completions/mean_length": 1000.1897583007812, "completions/mean_terminated_length": 390.8229217529297, "completions/min_length": 833.0, "completions/min_terminated_length": 321.0, "epoch": 0.009259950713165559, "grad_norm": 0.06064113602042198, "kl": 0.00035953521728515625, "learning_rate": 1.8750000000000003e-06, "loss": 0.0048, "num_tokens": 16380711.0, "reward": 0.2728794738650322, "reward_std": 0.017919870326295495, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.272879458963871, "rewards/tag_count_reward/std": 0.049723215866833925, "step": 31 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.984375, "completions/max_length": 1024.0, "completions/max_terminated_length": 461.25, "completions/mean_length": 1022.0044860839844, "completions/mean_terminated_length": 442.0416717529297, "completions/min_length": 921.75, "completions/min_terminated_length": 409.75, "epoch": 0.009558658800687028, "grad_norm": 0.046122368425130844, "kl": 0.00037097930908203125, "learning_rate": 1.9375e-06, "loss": 0.0012, "num_tokens": 16921817.0, "reward": 0.297433041036129, "reward_std": 0.02632934134453535, "rewards/accuracy_reward/mean": 0.0364583320915699, "rewards/accuracy_reward/std": 0.08869794011116028, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.266183041036129, "rewards/tag_count_reward/std": 0.03949749656021595, "step": 32 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9799107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 463.5, "completions/mean_length": 1019.6875152587891, "completions/mean_terminated_length": 407.66668701171875, "completions/min_length": 811.5, "completions/min_terminated_length": 299.5, "epoch": 0.009857366888208497, "grad_norm": 0.04437066614627838, "kl": 0.000392913818359375, "learning_rate": 2.0000000000000003e-06, "loss": 0.0029, "num_tokens": 17448525.0, "reward": 0.2918526977300644, "reward_std": 0.03104579751379788, "rewards/accuracy_reward/mean": 0.02678571455180645, "rewards/accuracy_reward/std": 0.07767121493816376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.265066958963871, "rewards/tag_count_reward/std": 0.055355607997626066, "step": 33 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9955357142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 475.0, "completions/mean_length": 1023.6696624755859, "completions/mean_terminated_length": 475.0, "completions/min_length": 987.0, "completions/min_terminated_length": 475.0, "epoch": 0.010156074975729968, "grad_norm": 0.05095032602548599, "kl": 0.0003829002380371094, "learning_rate": 2.0625e-06, "loss": 0.0003, "num_tokens": 17977897.0, "reward": 0.302455373108387, "reward_std": 0.07325492170639336, "rewards/accuracy_reward/mean": 0.04241071501746774, "rewards/accuracy_reward/std": 0.1656535044312477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2600446417927742, "rewards/tag_count_reward/std": 0.04508764902129769, "step": 34 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 498.0, "completions/mean_length": 1013.5625152587891, "completions/mean_terminated_length": 424.1470642089844, "completions/min_length": 836.75, "completions/min_terminated_length": 324.75, "epoch": 0.010454783063251438, "grad_norm": 0.05340949445962906, "kl": 0.0004467964172363281, "learning_rate": 2.125e-06, "loss": 0.0086, "num_tokens": 18510005.0, "reward": 0.3359375149011612, "reward_std": 0.059771391563117504, "rewards/accuracy_reward/mean": 0.06473214109428227, "rewards/accuracy_reward/std": 0.18958687596023083, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2712053582072258, "rewards/tag_count_reward/std": 0.0467427596449852, "step": 35 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 511.25, "completions/mean_length": 1022.8794860839844, "completions/mean_terminated_length": 484.0833435058594, "completions/min_length": 952.25, "completions/min_terminated_length": 440.25, "epoch": 0.010753491150772907, "grad_norm": 0.05239058658480644, "kl": 0.00046253204345703125, "learning_rate": 2.1875000000000002e-06, "loss": 0.0002, "num_tokens": 19041263.0, "reward": 0.3203125149011612, "reward_std": 0.0599199696443975, "rewards/accuracy_reward/mean": 0.046875000232830644, "rewards/accuracy_reward/std": 0.16321960650384426, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2734375, "rewards/tag_count_reward/std": 0.06622828310355544, "step": 36 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9642857142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 490.0, "completions/mean_length": 1018.3951110839844, "completions/mean_terminated_length": 454.28334045410156, "completions/min_length": 917.75, "completions/min_terminated_length": 405.75, "epoch": 0.011052199238294378, "grad_norm": 0.05852353572845459, "kl": 0.0004944801330566406, "learning_rate": 2.25e-06, "loss": 0.0027, "num_tokens": 19573168.0, "reward": 0.2801339402794838, "reward_std": 0.03298688982613385, "rewards/accuracy_reward/mean": 0.013392857275903225, "rewards/accuracy_reward/std": 0.056545503437519073, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2667410671710968, "rewards/tag_count_reward/std": 0.0446317377500236, "step": 37 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 244.25, "completions/mean_length": 1023.6183166503906, "completions/mean_terminated_length": 234.625, "completions/min_length": 993.0, "completions/min_terminated_length": 225.0, "epoch": 0.011350907325815847, "grad_norm": 0.03633173927664757, "kl": 0.0004825592041015625, "learning_rate": 2.3125000000000003e-06, "loss": 0.0006, "num_tokens": 20106821.0, "reward": 0.2550223246216774, "reward_std": 0.010105593595653772, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2550223246216774, "rewards/tag_count_reward/std": 0.024272767826914787, "step": 38 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9642857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 503.0, "completions/mean_length": 1017.0759124755859, "completions/mean_terminated_length": 416.3166809082031, "completions/min_length": 833.75, "completions/min_terminated_length": 321.75, "epoch": 0.011649615413337316, "grad_norm": 0.0466119684278965, "kl": 0.00055694580078125, "learning_rate": 2.375e-06, "loss": 0.0043, "num_tokens": 20633255.0, "reward": 0.313616082072258, "reward_std": 0.053534191101789474, "rewards/accuracy_reward/mean": 0.04910714365541935, "rewards/accuracy_reward/std": 0.14928054809570312, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2645089328289032, "rewards/tag_count_reward/std": 0.03987106867134571, "step": 39 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 475.75, "completions/mean_length": 1012.8884124755859, "completions/mean_terminated_length": 411.49375915527344, "completions/min_length": 835.75, "completions/min_terminated_length": 323.75, "epoch": 0.011948323500858785, "grad_norm": 0.06717411428689957, "kl": 0.000644683837890625, "learning_rate": 2.4375e-06, "loss": 0.0092, "num_tokens": 21164693.0, "reward": 0.3253348395228386, "reward_std": 0.08643224276602268, "rewards/accuracy_reward/mean": 0.04687500116415322, "rewards/accuracy_reward/std": 0.16596821323037148, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2784598246216774, "rewards/tag_count_reward/std": 0.06881650257855654, "step": 40 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 247.5, "completions/mean_length": 1021.638427734375, "completions/mean_terminated_length": 247.5, "completions/min_length": 759.5, "completions/min_terminated_length": 247.5, "epoch": 0.012247031588380256, "grad_norm": 0.045745257288217545, "kl": 0.0006799697875976562, "learning_rate": 2.5e-06, "loss": 0.0001, "num_tokens": 21702115.0, "reward": 0.2695312574505806, "reward_std": 0.04156872257590294, "rewards/accuracy_reward/mean": 0.008928571594879031, "rewards/accuracy_reward/std": 0.06416834704577923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2606026828289032, "rewards/tag_count_reward/std": 0.04154945630580187, "step": 41 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.012545739675901725, "grad_norm": 0.02383517287671566, "kl": 0.0006589889526367188, "learning_rate": 2.5625e-06, "loss": 0.0, "num_tokens": 22232867.0, "reward": 0.2572544664144516, "reward_std": 0.015709804370999336, "rewards/accuracy_reward/mean": 0.0066964286379516125, "rewards/accuracy_reward/std": 0.040545567870140076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2505580335855484, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 42 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 464.5, "completions/mean_length": 1022.4442138671875, "completions/mean_terminated_length": 424.875, "completions/min_length": 897.25, "completions/min_terminated_length": 385.25, "epoch": 0.012844447763423195, "grad_norm": 0.05925605818629265, "kl": 0.0007524490356445312, "learning_rate": 2.6250000000000003e-06, "loss": 0.0048, "num_tokens": 22765402.0, "reward": 0.2834821566939354, "reward_std": 0.06637045461684465, "rewards/accuracy_reward/mean": 0.017857143189758062, "rewards/accuracy_reward/std": 0.0924072265625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.265625, "rewards/tag_count_reward/std": 0.04966357909142971, "step": 43 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.5, "completions/mean_length": 1014.7433624267578, "completions/mean_terminated_length": 904.9943237304688, "completions/min_length": 807.75, "completions/min_terminated_length": 807.75, "epoch": 0.013143155850944664, "grad_norm": 0.06595340371131897, "kl": 0.0009508132934570312, "learning_rate": 2.6875e-06, "loss": 0.0054, "num_tokens": 23292215.0, "reward": 0.3303571566939354, "reward_std": 0.07576606562361121, "rewards/accuracy_reward/mean": 0.0491071417927742, "rewards/accuracy_reward/std": 0.14947086572647095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.28125, "rewards/tag_count_reward/std": 0.07603074703365564, "step": 44 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9977678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 219.5, "completions/mean_length": 1023.6741180419922, "completions/mean_terminated_length": 219.5, "completions/min_length": 987.5, "completions/min_terminated_length": 219.5, "epoch": 0.013441863938466135, "grad_norm": 0.0529695488512516, "kl": 0.0009050369262695312, "learning_rate": 2.7500000000000004e-06, "loss": 0.0002, "num_tokens": 23824773.0, "reward": 0.2667410895228386, "reward_std": 0.019680705852806568, "rewards/accuracy_reward/mean": 0.0, "rewards/accuracy_reward/std": 0.0, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2667410671710968, "rewards/tag_count_reward/std": 0.051716179586946964, "step": 45 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 1.0, "completions/max_length": 1024.0, "completions/max_terminated_length": 0.0, "completions/mean_length": 1024.0, "completions/mean_terminated_length": 0.0, "completions/min_length": 1024.0, "completions/min_terminated_length": 0.0, "epoch": 0.013740572025987604, "grad_norm": 0.057629313319921494, "kl": 0.0009765625, "learning_rate": 2.8125e-06, "loss": 0.0, "num_tokens": 24356981.0, "reward": 0.2606026902794838, "reward_std": 0.03533772728405893, "rewards/accuracy_reward/mean": 0.0066964286379516125, "rewards/accuracy_reward/std": 0.040545567870140076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.25390625, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 46 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 444.75, "completions/mean_length": 1016.9397583007812, "completions/mean_terminated_length": 416.4250030517578, "completions/min_length": 891.5, "completions/min_terminated_length": 379.5, "epoch": 0.014039280113509073, "grad_norm": 0.06135973706841469, "kl": 0.001293182373046875, "learning_rate": 2.875e-06, "loss": 0.0032, "num_tokens": 24881642.0, "reward": 0.3041294738650322, "reward_std": 0.05731133185327053, "rewards/accuracy_reward/mean": 0.02678571455180645, "rewards/accuracy_reward/std": 0.07767121493816376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.27734375, "rewards/tag_count_reward/std": 0.05960681103169918, "step": 47 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 660.5, "completions/mean_length": 1007.5870819091797, "completions/mean_terminated_length": 560.296875, "completions/min_length": 757.0, "completions/min_terminated_length": 501.0, "epoch": 0.014337988201030542, "grad_norm": 0.06973817199468613, "kl": 0.0012969970703125, "learning_rate": 2.9375000000000003e-06, "loss": 0.0092, "num_tokens": 25411649.0, "reward": 0.3320312649011612, "reward_std": 0.06988973217085004, "rewards/accuracy_reward/mean": 0.06473214249126613, "rewards/accuracy_reward/std": 0.1687364336103201, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2672991156578064, "rewards/tag_count_reward/std": 0.05394811276346445, "step": 48 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 168.0, "completions/mean_length": 1006.5959930419922, "completions/mean_terminated_length": 134.171875, "completions/min_length": 850.25, "completions/min_terminated_length": 82.25, "epoch": 0.014636696288552013, "grad_norm": 0.03178580477833748, "kl": 0.001453399658203125, "learning_rate": 3e-06, "loss": 0.0001, "num_tokens": 25930732.0, "reward": 0.2756696492433548, "reward_std": 0.03253185749053955, "rewards/accuracy_reward/mean": 0.013392857508733869, "rewards/accuracy_reward/std": 0.07548443786799908, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2622767835855484, "rewards/tag_count_reward/std": 0.030409857165068388, "step": 49 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 743.25, "completions/mean_length": 1017.8460083007812, "completions/mean_terminated_length": 689.6389007568359, "completions/min_length": 875.5, "completions/min_terminated_length": 619.5, "epoch": 0.014935404376073482, "grad_norm": 0.052705325186252594, "kl": 0.0017299652099609375, "learning_rate": 3.0625000000000003e-06, "loss": 0.0009, "num_tokens": 26454615.0, "reward": 0.3498884066939354, "reward_std": 0.05624916963279247, "rewards/accuracy_reward/mean": 0.0647321417927742, "rewards/accuracy_reward/std": 0.1685778871178627, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.28515625, "rewards/tag_count_reward/std": 0.07271651178598404, "step": 50 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9419642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 749.5, "completions/mean_length": 1011.4977874755859, "completions/mean_terminated_length": 682.3863677978516, "completions/min_length": 874.75, "completions/min_terminated_length": 618.75, "epoch": 0.015234112463594952, "grad_norm": 0.07413821667432785, "kl": 0.002044677734375, "learning_rate": 3.125e-06, "loss": 0.0044, "num_tokens": 26986774.0, "reward": 0.317522332072258, "reward_std": 0.04384439159184694, "rewards/accuracy_reward/mean": 0.0357142873108387, "rewards/accuracy_reward/std": 0.08787495642900467, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.281808041036129, "rewards/tag_count_reward/std": 0.0670435531064868, "step": 51 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 746.0, "completions/mean_length": 1011.8214569091797, "completions/mean_terminated_length": 669.6666870117188, "completions/min_length": 847.75, "completions/min_terminated_length": 591.75, "epoch": 0.01553282055111642, "grad_norm": 0.05609305575489998, "kl": 0.00196075439453125, "learning_rate": 3.1875e-06, "loss": 0.0022, "num_tokens": 27509174.0, "reward": 0.3231026902794838, "reward_std": 0.060010447166860104, "rewards/accuracy_reward/mean": 0.046875, "rewards/accuracy_reward/std": 0.14511175453662872, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2762276753783226, "rewards/tag_count_reward/std": 0.06500177644193172, "step": 52 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 750.25, "completions/mean_length": 1022.3236846923828, "completions/mean_terminated_length": 718.6666717529297, "completions/min_length": 934.75, "completions/min_terminated_length": 678.75, "epoch": 0.01583152863863789, "grad_norm": 0.04886524751782417, "kl": 0.0019474029541015625, "learning_rate": 3.2500000000000002e-06, "loss": 0.0016, "num_tokens": 28035831.0, "reward": 0.2935268059372902, "reward_std": 0.05057145212776959, "rewards/accuracy_reward/mean": 0.026785714086145163, "rewards/accuracy_reward/std": 0.10881233960390091, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2667410671710968, "rewards/tag_count_reward/std": 0.05351166380569339, "step": 53 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 1024.0, "completions/max_terminated_length": 495.5, "completions/mean_length": 1005.1652069091797, "completions/mean_terminated_length": 361.3214569091797, "completions/min_length": 758.5, "completions/min_terminated_length": 246.5, "epoch": 0.01613023672615936, "grad_norm": 0.05488144978880882, "kl": 0.0023441314697265625, "learning_rate": 3.3125e-06, "loss": 0.0027, "num_tokens": 28556769.0, "reward": 0.3069196566939354, "reward_std": 0.025544091360643506, "rewards/accuracy_reward/mean": 0.03125, "rewards/accuracy_reward/std": 0.08305132389068604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2756696417927742, "rewards/tag_count_reward/std": 0.055375372525304556, "step": 54 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 488.5, "completions/mean_length": 1020.2053833007812, "completions/mean_terminated_length": 430.4583282470703, "completions/min_length": 866.5, "completions/min_terminated_length": 354.5, "epoch": 0.016428944813680832, "grad_norm": 0.04348389431834221, "kl": 0.0021381378173828125, "learning_rate": 3.3750000000000003e-06, "loss": 0.0016, "num_tokens": 29089885.0, "reward": 0.297991082072258, "reward_std": 0.024984613060951233, "rewards/accuracy_reward/mean": 0.02901785634458065, "rewards/accuracy_reward/std": 0.08043753355741501, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.268973208963871, "rewards/tag_count_reward/std": 0.044944485649466515, "step": 55 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.921875, "completions/max_length": 1024.0, "completions/max_terminated_length": 694.25, "completions/mean_length": 988.2656555175781, "completions/mean_terminated_length": 512.8541641235352, "completions/min_length": 655.5, "completions/min_terminated_length": 399.5, "epoch": 0.0167276529012023, "grad_norm": 0.05108587071299553, "kl": 0.003658294677734375, "learning_rate": 3.4375e-06, "loss": 0.0015, "num_tokens": 29602516.0, "reward": 0.3671875223517418, "reward_std": 0.08528654929250479, "rewards/accuracy_reward/mean": 0.08258928777649999, "rewards/accuracy_reward/std": 0.2053450234234333, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2845982164144516, "rewards/tag_count_reward/std": 0.0708478894084692, "step": 56 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 942.25, "completions/mean_length": 1003.4955749511719, "completions/mean_terminated_length": 830.2148132324219, "completions/min_length": 721.25, "completions/min_terminated_length": 721.25, "epoch": 0.01702636098872377, "grad_norm": 0.06546254456043243, "kl": 0.002956390380859375, "learning_rate": 3.5e-06, "loss": 0.0047, "num_tokens": 30117106.0, "reward": 0.3577009066939354, "reward_std": 0.06603909283876419, "rewards/accuracy_reward/mean": 0.06696428544819355, "rewards/accuracy_reward/std": 0.17054396867752075, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2907366082072258, "rewards/tag_count_reward/std": 0.08911363501101732, "step": 57 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 506.25, "completions/mean_length": 1023.0089416503906, "completions/mean_terminated_length": 480.625, "completions/min_length": 955.5, "completions/min_terminated_length": 443.5, "epoch": 0.017325069076245238, "grad_norm": 0.07222483307123184, "kl": 0.00257110595703125, "learning_rate": 3.5625e-06, "loss": 0.0007, "num_tokens": 30650166.0, "reward": 0.3828125149011612, "reward_std": 0.1297378782182932, "rewards/accuracy_reward/mean": 0.10267857206054032, "rewards/accuracy_reward/std": 0.27140951715409756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2801339253783226, "rewards/tag_count_reward/std": 0.07582549378275871, "step": 58 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 893.75, "completions/mean_length": 993.4308471679688, "completions/mean_terminated_length": 724.5792694091797, "completions/min_length": 509.0, "completions/min_terminated_length": 509.0, "epoch": 0.01762377716376671, "grad_norm": 0.08538337796926498, "kl": 0.003017425537109375, "learning_rate": 3.625e-06, "loss": 0.006, "num_tokens": 31165063.0, "reward": 0.3264509066939354, "reward_std": 0.10809027217328548, "rewards/accuracy_reward/mean": 0.029017858440056443, "rewards/accuracy_reward/std": 0.15481065586209297, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2974330335855484, "rewards/tag_count_reward/std": 0.09402106516063213, "step": 59 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 1024.0, "completions/max_terminated_length": 467.75, "completions/mean_length": 992.9308319091797, "completions/mean_terminated_length": 251.953125, "completions/min_length": 626.0, "completions/min_terminated_length": 114.0, "epoch": 0.01792248525128818, "grad_norm": 0.05558789521455765, "kl": 0.002460479736328125, "learning_rate": 3.6875000000000007e-06, "loss": 0.0125, "num_tokens": 31684584.0, "reward": 0.279017873108387, "reward_std": 0.04191167652606964, "rewards/accuracy_reward/mean": 0.01116071455180645, "rewards/accuracy_reward/std": 0.073802400380373, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2678571417927742, "rewards/tag_count_reward/std": 0.05367471091449261, "step": 60 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 978.5, "completions/mean_length": 996.9241638183594, "completions/mean_terminated_length": 799.8611145019531, "completions/min_length": 629.5, "completions/min_terminated_length": 629.5, "epoch": 0.018221193338809647, "grad_norm": 0.08840494602918625, "kl": 0.00322723388671875, "learning_rate": 3.7500000000000005e-06, "loss": 0.0028, "num_tokens": 32202086.0, "reward": 0.3934151977300644, "reward_std": 0.1191147230565548, "rewards/accuracy_reward/mean": 0.1026785708963871, "rewards/accuracy_reward/std": 0.2981107607483864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2907366007566452, "rewards/tag_count_reward/std": 0.09170791134238243, "step": 61 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 734.5, "completions/mean_length": 988.1741333007812, "completions/mean_terminated_length": 575.4889831542969, "completions/min_length": 684.5, "completions/min_terminated_length": 428.5, "epoch": 0.018519901426331118, "grad_norm": 0.08759739995002747, "kl": 0.003200531005859375, "learning_rate": 3.8125e-06, "loss": 0.0108, "num_tokens": 32721108.0, "reward": 0.326450914144516, "reward_std": 0.06441474077291787, "rewards/accuracy_reward/mean": 0.03348214225843549, "rewards/accuracy_reward/std": 0.11369437351822853, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.29296875, "rewards/tag_count_reward/std": 0.0892208619043231, "step": 62 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 490.75, "completions/mean_length": 1003.1629638671875, "completions/mean_terminated_length": 364.54412841796875, "completions/min_length": 775.0, "completions/min_terminated_length": 263.0, "epoch": 0.01881860951385259, "grad_norm": 0.062408287078142166, "kl": 0.003021240234375, "learning_rate": 3.875e-06, "loss": 0.004, "num_tokens": 33237229.0, "reward": 0.2818080484867096, "reward_std": 0.03931350912898779, "rewards/accuracy_reward/mean": 0.004464285913854837, "rewards/accuracy_reward/std": 0.047245556488633156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.27734375, "rewards/tag_count_reward/std": 0.06293313857167959, "step": 63 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 762.5, "completions/mean_length": 1010.1942291259766, "completions/mean_terminated_length": 651.5401458740234, "completions/min_length": 768.5, "completions/min_terminated_length": 512.5, "epoch": 0.019117317601374056, "grad_norm": 0.07943255454301834, "kl": 0.00373077392578125, "learning_rate": 3.9375e-06, "loss": 0.0082, "num_tokens": 33763860.0, "reward": 0.4263393059372902, "reward_std": 0.1191812101751566, "rewards/accuracy_reward/mean": 0.10714285564608872, "rewards/accuracy_reward/std": 0.28251451067626476, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3191964253783226, "rewards/tag_count_reward/std": 0.09160909755155444, "step": 64 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9419642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 954.0, "completions/mean_length": 1014.591552734375, "completions/mean_terminated_length": 884.3277893066406, "completions/min_length": 772.75, "completions/min_terminated_length": 772.75, "epoch": 0.019416025688895527, "grad_norm": 0.0479339100420475, "kl": 0.002849578857421875, "learning_rate": 4.000000000000001e-06, "loss": 0.001, "num_tokens": 34289613.0, "reward": 0.3018973395228386, "reward_std": 0.049580322578549385, "rewards/accuracy_reward/mean": 0.024553571827709675, "rewards/accuracy_reward/std": 0.108407162129879, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.27734375, "rewards/tag_count_reward/std": 0.07375888247042894, "step": 65 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9553571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 502.5, "completions/mean_length": 1017.0268096923828, "completions/mean_terminated_length": 445.0333557128906, "completions/min_length": 894.0, "completions/min_terminated_length": 382.0, "epoch": 0.019714733776416995, "grad_norm": 0.039794377982616425, "kl": 0.0026378631591796875, "learning_rate": 4.0625000000000005e-06, "loss": 0.0014, "num_tokens": 34830665.0, "reward": 0.277901791036129, "reward_std": 0.027838345617055893, "rewards/accuracy_reward/mean": 0.008928571827709675, "rewards/accuracy_reward/std": 0.04660273343324661, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2689732164144516, "rewards/tag_count_reward/std": 0.054382250644266605, "step": 66 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 737.5, "completions/mean_length": 1015.3906555175781, "completions/mean_terminated_length": 686.2166748046875, "completions/min_length": 900.25, "completions/min_terminated_length": 644.25, "epoch": 0.020013441863938466, "grad_norm": 0.0589945986866951, "kl": 0.003063201904296875, "learning_rate": 4.125e-06, "loss": 0.0006, "num_tokens": 35355672.0, "reward": 0.3007812574505806, "reward_std": 0.07964713126420975, "rewards/accuracy_reward/mean": 0.029017858440056443, "rewards/accuracy_reward/std": 0.13489972613751888, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2717633992433548, "rewards/tag_count_reward/std": 0.05478710774332285, "step": 67 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 1009.8013763427734, "completions/mean_terminated_length": 907.9445953369141, "completions/min_length": 802.75, "completions/min_terminated_length": 802.75, "epoch": 0.020312149951459937, "grad_norm": 0.07306487113237381, "kl": 0.004199981689453125, "learning_rate": 4.1875e-06, "loss": 0.0022, "num_tokens": 35879903.0, "reward": 0.3660714402794838, "reward_std": 0.11265680380165577, "rewards/accuracy_reward/mean": 0.0580357126891613, "rewards/accuracy_reward/std": 0.16026198863983154, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3080357238650322, "rewards/tag_count_reward/std": 0.1038170475512743, "step": 68 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 1002.9620971679688, "completions/mean_terminated_length": 860.579833984375, "completions/min_length": 702.0, "completions/min_terminated_length": 702.0, "epoch": 0.020610858038981404, "grad_norm": 0.0847926065325737, "kl": 0.004123687744140625, "learning_rate": 4.25e-06, "loss": 0.0064, "num_tokens": 36405982.0, "reward": 0.360491082072258, "reward_std": 0.07113150274381042, "rewards/accuracy_reward/mean": 0.05133928544819355, "rewards/accuracy_reward/std": 0.1502007693052292, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.309151791036129, "rewards/tag_count_reward/std": 0.09233940951526165, "step": 69 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.0, "completions/mean_length": 1016.5580749511719, "completions/mean_terminated_length": 838.3479309082031, "completions/min_length": 732.0, "completions/min_terminated_length": 732.0, "epoch": 0.020909566126502875, "grad_norm": 0.08152010291814804, "kl": 0.00438690185546875, "learning_rate": 4.312500000000001e-06, "loss": 0.0073, "num_tokens": 36930296.0, "reward": 0.4369419887661934, "reward_std": 0.17016438022255898, "rewards/accuracy_reward/mean": 0.1383928577415645, "rewards/accuracy_reward/std": 0.32129356265068054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2985491082072258, "rewards/tag_count_reward/std": 0.09550981223583221, "step": 70 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 746.25, "completions/mean_length": 1004.2299499511719, "completions/mean_terminated_length": 649.9687652587891, "completions/min_length": 781.0, "completions/min_terminated_length": 525.0, "epoch": 0.021208274214024346, "grad_norm": 0.08273611217737198, "kl": 0.00554656982421875, "learning_rate": 4.3750000000000005e-06, "loss": 0.0049, "num_tokens": 37450271.0, "reward": 0.400111623108387, "reward_std": 0.12536026095040143, "rewards/accuracy_reward/mean": 0.08035714365541935, "rewards/accuracy_reward/std": 0.22851300239562988, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3197544664144516, "rewards/tag_count_reward/std": 0.0893968278542161, "step": 71 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.5, "completions/mean_length": 981.6473541259766, "completions/mean_terminated_length": 756.3416061401367, "completions/min_length": 565.5, "completions/min_terminated_length": 565.5, "epoch": 0.021506982301545814, "grad_norm": 0.07782959192991257, "kl": 0.00499725341796875, "learning_rate": 4.4375e-06, "loss": 0.0041, "num_tokens": 37968369.0, "reward": 0.4263393059372902, "reward_std": 0.0996090043336153, "rewards/accuracy_reward/mean": 0.11383928405120969, "rewards/accuracy_reward/std": 0.29299789294600487, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3124999925494194, "rewards/tag_count_reward/std": 0.10560555942356586, "step": 72 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 962.5, "completions/mean_length": 962.1585235595703, "completions/mean_terminated_length": 767.7887573242188, "completions/min_length": 560.0, "completions/min_terminated_length": 560.0, "epoch": 0.021805690389067284, "grad_norm": 0.08673243969678879, "kl": 0.00623321533203125, "learning_rate": 4.5e-06, "loss": 0.0112, "num_tokens": 38473080.0, "reward": 0.4843750223517418, "reward_std": 0.08116957172751427, "rewards/accuracy_reward/mean": 0.1540178619325161, "rewards/accuracy_reward/std": 0.2905626595020294, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3303571417927742, "rewards/tag_count_reward/std": 0.10644344240427017, "step": 73 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 736.5, "completions/mean_length": 1003.7120819091797, "completions/mean_terminated_length": 657.9357147216797, "completions/min_length": 761.5, "completions/min_terminated_length": 505.5, "epoch": 0.022104398476588755, "grad_norm": 0.08560993522405624, "kl": 0.00539398193359375, "learning_rate": 4.5625e-06, "loss": -0.0006, "num_tokens": 38998103.0, "reward": 0.3794643059372902, "reward_std": 0.14529736153781414, "rewards/accuracy_reward/mean": 0.06473214318975806, "rewards/accuracy_reward/std": 0.20126811414957047, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3147321417927742, "rewards/tag_count_reward/std": 0.10747164860367775, "step": 74 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 929.5, "completions/mean_length": 983.9442443847656, "completions/mean_terminated_length": 756.2672500610352, "completions/min_length": 630.25, "completions/min_terminated_length": 630.25, "epoch": 0.022403106564110223, "grad_norm": 0.06736253201961517, "kl": 0.0055389404296875, "learning_rate": 4.625000000000001e-06, "loss": 0.0046, "num_tokens": 39512990.0, "reward": 0.3839285895228386, "reward_std": 0.05647896463051438, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.18233589082956314, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3058035671710968, "rewards/tag_count_reward/std": 0.10377917811274529, "step": 75 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 623.25, "completions/mean_length": 976.0870819091797, "completions/mean_terminated_length": 467.39109802246094, "completions/min_length": 576.75, "completions/min_terminated_length": 320.75, "epoch": 0.022701814651631694, "grad_norm": 0.0651644766330719, "kl": 0.0055694580078125, "learning_rate": 4.6875000000000004e-06, "loss": 0.003, "num_tokens": 40016037.0, "reward": 0.3716518059372902, "reward_std": 0.055599984247237444, "rewards/accuracy_reward/mean": 0.07589285774156451, "rewards/accuracy_reward/std": 0.21394708007574081, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2957589328289032, "rewards/tag_count_reward/std": 0.08858883380889893, "step": 76 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 729.75, "completions/mean_length": 992.2678985595703, "completions/mean_terminated_length": 601.9909515380859, "completions/min_length": 732.0, "completions/min_terminated_length": 476.0, "epoch": 0.02300052273915316, "grad_norm": 0.07874111831188202, "kl": 0.005451202392578125, "learning_rate": 4.75e-06, "loss": 0.007, "num_tokens": 40528941.0, "reward": 0.3934151977300644, "reward_std": 0.08897749334573746, "rewards/accuracy_reward/mean": 0.09598214295692742, "rewards/accuracy_reward/std": 0.21541381254792213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.297433041036129, "rewards/tag_count_reward/std": 0.0903292428702116, "step": 77 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9732142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 435.75, "completions/mean_length": 1018.8906555175781, "completions/mean_terminated_length": 390.6591033935547, "completions/min_length": 845.5, "completions/min_terminated_length": 333.5, "epoch": 0.023299230826674632, "grad_norm": 0.07176060974597931, "kl": 0.004070281982421875, "learning_rate": 4.8125e-06, "loss": 0.0026, "num_tokens": 41057452.0, "reward": 0.3158482313156128, "reward_std": 0.07340040430426598, "rewards/accuracy_reward/mean": 0.044642857275903225, "rewards/accuracy_reward/std": 0.1395968273282051, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2712053507566452, "rewards/tag_count_reward/std": 0.06914053857326508, "step": 78 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 729.5, "completions/mean_length": 999.7589569091797, "completions/mean_terminated_length": 611.9604187011719, "completions/min_length": 713.75, "completions/min_terminated_length": 457.75, "epoch": 0.023597938914196103, "grad_norm": 0.08062504231929779, "kl": 0.004383087158203125, "learning_rate": 4.875e-06, "loss": 0.0234, "num_tokens": 41584640.0, "reward": 0.3007812649011612, "reward_std": 0.06233404204249382, "rewards/accuracy_reward/mean": 0.01785714365541935, "rewards/accuracy_reward/std": 0.06467421352863312, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2829241007566452, "rewards/tag_count_reward/std": 0.08061835542321205, "step": 79 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 1017.7053985595703, "completions/mean_terminated_length": 888.7142944335938, "completions/min_length": 788.75, "completions/min_terminated_length": 788.75, "epoch": 0.02389664700171757, "grad_norm": 0.08515530079603195, "kl": 0.005340576171875, "learning_rate": 4.937500000000001e-06, "loss": 0.0057, "num_tokens": 42116636.0, "reward": 0.3286830484867096, "reward_std": 0.09006298519670963, "rewards/accuracy_reward/mean": 0.035714286379516125, "rewards/accuracy_reward/std": 0.12427395582199097, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2929687425494194, "rewards/tag_count_reward/std": 0.08653074875473976, "step": 80 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.921875, "completions/max_length": 1024.0, "completions/max_terminated_length": 719.25, "completions/mean_length": 1001.9933471679688, "completions/mean_terminated_length": 578.7447967529297, "completions/min_length": 667.5, "completions/min_terminated_length": 411.5, "epoch": 0.02419535508923904, "grad_norm": 0.08054371178150177, "kl": 0.00640869140625, "learning_rate": 5e-06, "loss": 0.0033, "num_tokens": 42636313.0, "reward": 0.3850446566939354, "reward_std": 0.09301280695945024, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.18212634325027466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3069196417927742, "rewards/tag_count_reward/std": 0.10097978450357914, "step": 81 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 747.0, "completions/mean_length": 1011.9866485595703, "completions/mean_terminated_length": 648.3625030517578, "completions/min_length": 791.75, "completions/min_terminated_length": 535.75, "epoch": 0.024494063176760512, "grad_norm": 0.08093322813510895, "kl": 0.00670623779296875, "learning_rate": 5.0625e-06, "loss": 0.0096, "num_tokens": 43170931.0, "reward": 0.3370535895228386, "reward_std": 0.06958008860237896, "rewards/accuracy_reward/mean": 0.0334821417927742, "rewards/accuracy_reward/std": 0.08552655577659607, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3035714253783226, "rewards/tag_count_reward/std": 0.10072925500571728, "step": 82 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.5, "completions/mean_length": 971.6786041259766, "completions/mean_terminated_length": 744.817138671875, "completions/min_length": 547.0, "completions/min_terminated_length": 547.0, "epoch": 0.02479277126428198, "grad_norm": 0.08012297749519348, "kl": 0.007843017578125, "learning_rate": 5.125e-06, "loss": 0.0094, "num_tokens": 43688835.0, "reward": 0.373883955180645, "reward_std": 0.07563094794750214, "rewards/accuracy_reward/mean": 0.046874999068677425, "rewards/accuracy_reward/std": 0.14207205921411514, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3270089253783226, "rewards/tag_count_reward/std": 0.11307711526751518, "step": 83 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 994.8393249511719, "completions/mean_terminated_length": 841.8412933349609, "completions/min_length": 667.0, "completions/min_terminated_length": 667.0, "epoch": 0.02509147935180345, "grad_norm": 0.09175465255975723, "kl": 0.00762176513671875, "learning_rate": 5.187500000000001e-06, "loss": 0.0026, "num_tokens": 44206491.0, "reward": 0.3822544813156128, "reward_std": 0.11552051967009902, "rewards/accuracy_reward/mean": 0.07142857369035482, "rewards/accuracy_reward/std": 0.2160297855734825, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3108258843421936, "rewards/tag_count_reward/std": 0.10670643113553524, "step": 84 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 1024.0, "completions/max_terminated_length": 760.75, "completions/mean_length": 977.3326263427734, "completions/mean_terminated_length": 598.5619964599609, "completions/min_length": 700.25, "completions/min_terminated_length": 444.25, "epoch": 0.02539018743932492, "grad_norm": 0.08394108712673187, "kl": 0.009185791015625, "learning_rate": 5.2500000000000006e-06, "loss": 0.0057, "num_tokens": 44712416.0, "reward": 0.4441964402794838, "reward_std": 0.09175788168795407, "rewards/accuracy_reward/mean": 0.11830357415601611, "rewards/accuracy_reward/std": 0.24622111022472382, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3258928582072258, "rewards/tag_count_reward/std": 0.09964521508663893, "step": 85 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 965.75, "completions/mean_length": 992.3884429931641, "completions/mean_terminated_length": 749.6417999267578, "completions/min_length": 522.75, "completions/min_terminated_length": 522.75, "epoch": 0.02568889552684639, "grad_norm": 0.08988907188177109, "kl": 0.00832366943359375, "learning_rate": 5.3125e-06, "loss": 0.0209, "num_tokens": 45235662.0, "reward": 0.4174107387661934, "reward_std": 0.12120191752910614, "rewards/accuracy_reward/mean": 0.10491071548312902, "rewards/accuracy_reward/std": 0.3013125881552696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3125, "rewards/tag_count_reward/std": 0.10847658291459084, "step": 86 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 939.5, "completions/mean_length": 979.2545013427734, "completions/mean_terminated_length": 769.9906311035156, "completions/min_length": 621.25, "completions/min_terminated_length": 621.25, "epoch": 0.02598760361436786, "grad_norm": 0.08180143684148788, "kl": 0.01042938232421875, "learning_rate": 5.375e-06, "loss": 0.0043, "num_tokens": 45738480.0, "reward": 0.4492187649011612, "reward_std": 0.15252777375280857, "rewards/accuracy_reward/mean": 0.12499999976716936, "rewards/accuracy_reward/std": 0.2695323470979929, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.32421875, "rewards/tag_count_reward/std": 0.1047529960051179, "step": 87 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9107142857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 675.5, "completions/mean_length": 1000.0402069091797, "completions/mean_terminated_length": 558.6852874755859, "completions/min_length": 686.75, "completions/min_terminated_length": 430.75, "epoch": 0.026286311701889328, "grad_norm": 0.07833461463451385, "kl": 0.00865936279296875, "learning_rate": 5.4375e-06, "loss": 0.0079, "num_tokens": 46258130.0, "reward": 0.3476562723517418, "reward_std": 0.10576667450368404, "rewards/accuracy_reward/mean": 0.053571428172290325, "rewards/accuracy_reward/std": 0.20593168959021568, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2940848246216774, "rewards/tag_count_reward/std": 0.09200722817331553, "step": 88 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 995.5625457763672, "completions/mean_terminated_length": 848.2879943847656, "completions/min_length": 623.5, "completions/min_terminated_length": 623.5, "epoch": 0.0265850197894108, "grad_norm": 0.08255854994058609, "kl": 0.010955810546875, "learning_rate": 5.500000000000001e-06, "loss": 0.0036, "num_tokens": 46780462.0, "reward": 0.4185268059372902, "reward_std": 0.1166799757629633, "rewards/accuracy_reward/mean": 0.10156250046566129, "rewards/accuracy_reward/std": 0.25889817625284195, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3270089328289032, "rewards/tag_count_reward/std": 0.11497674323618412, "step": 89 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8816964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.25, "completions/mean_length": 989.7857513427734, "completions/mean_terminated_length": 810.2250213623047, "completions/min_length": 565.0, "completions/min_terminated_length": 565.0, "epoch": 0.02688372787693227, "grad_norm": 0.0882234126329422, "kl": 0.010986328125, "learning_rate": 5.5625000000000005e-06, "loss": -0.0003, "num_tokens": 47301278.0, "reward": 0.3722098395228386, "reward_std": 0.11953129805624485, "rewards/accuracy_reward/mean": 0.06026785750873387, "rewards/accuracy_reward/std": 0.21932310797274113, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.311941958963871, "rewards/tag_count_reward/std": 0.10603687353432178, "step": 90 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8080357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.5, "completions/mean_length": 963.3437957763672, "completions/mean_terminated_length": 728.8751983642578, "completions/min_length": 513.5, "completions/min_terminated_length": 513.5, "epoch": 0.027182435964453737, "grad_norm": 0.07671667635440826, "kl": 0.01214599609375, "learning_rate": 5.625e-06, "loss": 0.0072, "num_tokens": 47804456.0, "reward": 0.384486623108387, "reward_std": 0.09358065202832222, "rewards/accuracy_reward/mean": 0.05357143050059676, "rewards/accuracy_reward/std": 0.18028218299150467, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3309151753783226, "rewards/tag_count_reward/std": 0.11721918918192387, "step": 91 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8258928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 984.3571929931641, "completions/mean_terminated_length": 805.8433074951172, "completions/min_length": 624.5, "completions/min_terminated_length": 624.5, "epoch": 0.027481144051975208, "grad_norm": 0.09284783154726028, "kl": 0.0122833251953125, "learning_rate": 5.6875e-06, "loss": 0.0093, "num_tokens": 48324424.0, "reward": 0.4330357313156128, "reward_std": 0.11909635085612535, "rewards/accuracy_reward/mean": 0.09821428544819355, "rewards/accuracy_reward/std": 0.2499719262123108, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3348214328289032, "rewards/tag_count_reward/std": 0.11803762428462505, "step": 92 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 970.7567443847656, "completions/mean_terminated_length": 816.8922119140625, "completions/min_length": 637.25, "completions/min_terminated_length": 637.25, "epoch": 0.027779852139496675, "grad_norm": 0.09242645651102066, "kl": 0.0133819580078125, "learning_rate": 5.75e-06, "loss": 0.0062, "num_tokens": 48827803.0, "reward": 0.5066964477300644, "reward_std": 0.14125356450676918, "rewards/accuracy_reward/mean": 0.16517857182770967, "rewards/accuracy_reward/std": 0.3497554361820221, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3415178507566452, "rewards/tag_count_reward/std": 0.11691479198634624, "step": 93 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8995535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 927.25, "completions/mean_length": 1001.6250457763672, "completions/mean_terminated_length": 825.5120544433594, "completions/min_length": 705.5, "completions/min_terminated_length": 705.5, "epoch": 0.028078560227018146, "grad_norm": 0.10701184719800949, "kl": 0.0116119384765625, "learning_rate": 5.812500000000001e-06, "loss": 0.016, "num_tokens": 49350547.0, "reward": 0.381138414144516, "reward_std": 0.14232603833079338, "rewards/accuracy_reward/mean": 0.06696428661234677, "rewards/accuracy_reward/std": 0.23340874537825584, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3141741007566452, "rewards/tag_count_reward/std": 0.10463365726172924, "step": 94 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 994.7232513427734, "completions/mean_terminated_length": 864.5555725097656, "completions/min_length": 697.25, "completions/min_terminated_length": 697.25, "epoch": 0.028377268314539617, "grad_norm": 0.07837118208408356, "kl": 0.00995635986328125, "learning_rate": 5.8750000000000005e-06, "loss": 0.0052, "num_tokens": 49863639.0, "reward": 0.3309151977300644, "reward_std": 0.07525985315442085, "rewards/accuracy_reward/mean": 0.033482143422588706, "rewards/accuracy_reward/std": 0.13455083407461643, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2974330335855484, "rewards/tag_count_reward/std": 0.08724715560674667, "step": 95 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 937.0, "completions/mean_length": 979.5089721679688, "completions/mean_terminated_length": 745.4316558837891, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "epoch": 0.028675976402061085, "grad_norm": 0.09178336709737778, "kl": 0.01151275634765625, "learning_rate": 5.9375e-06, "loss": 0.0172, "num_tokens": 50373595.0, "reward": 0.4095982313156128, "reward_std": 0.12030867207795382, "rewards/accuracy_reward/mean": 0.08928571455180645, "rewards/accuracy_reward/std": 0.24304793030023575, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3203125, "rewards/tag_count_reward/std": 0.10369388200342655, "step": 96 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.75, "completions/mean_length": 990.0067443847656, "completions/mean_terminated_length": 819.3621978759766, "completions/min_length": 661.0, "completions/min_terminated_length": 661.0, "epoch": 0.028974684489582556, "grad_norm": 0.07378272712230682, "kl": 0.01092529296875, "learning_rate": 6e-06, "loss": 0.0069, "num_tokens": 50888734.0, "reward": 0.3521205484867096, "reward_std": 0.06455331854522228, "rewards/accuracy_reward/mean": 0.044642857974395156, "rewards/accuracy_reward/std": 0.14698603935539722, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3074776753783226, "rewards/tag_count_reward/std": 0.09764310158789158, "step": 97 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.0, "completions/mean_length": 993.654052734375, "completions/mean_terminated_length": 843.7456359863281, "completions/min_length": 684.25, "completions/min_terminated_length": 684.25, "epoch": 0.029273392577104027, "grad_norm": 0.07864398509263992, "kl": 0.01004791259765625, "learning_rate": 6.0625e-06, "loss": 0.0103, "num_tokens": 51412931.0, "reward": 0.4570312723517418, "reward_std": 0.0877478364855051, "rewards/accuracy_reward/mean": 0.14732143026776612, "rewards/accuracy_reward/std": 0.24141780845820904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3097098171710968, "rewards/tag_count_reward/std": 0.09509929362684488, "step": 98 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7790178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 974.3995971679688, "completions/mean_terminated_length": 816.790771484375, "completions/min_length": 575.0, "completions/min_terminated_length": 575.0, "epoch": 0.029572100664625494, "grad_norm": 0.08460793644189835, "kl": 0.012298583984375, "learning_rate": 6.125000000000001e-06, "loss": 0.0104, "num_tokens": 51923382.0, "reward": 0.4375000298023224, "reward_std": 0.0703044505789876, "rewards/accuracy_reward/mean": 0.09151786123402417, "rewards/accuracy_reward/std": 0.20174307003617287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3459821417927742, "rewards/tag_count_reward/std": 0.1187976822257042, "step": 99 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.25, "completions/mean_length": 972.8058624267578, "completions/mean_terminated_length": 742.48779296875, "completions/min_length": 499.75, "completions/min_terminated_length": 499.75, "epoch": 0.029870808752146965, "grad_norm": 0.09248195588588715, "kl": 0.011688232421875, "learning_rate": 6.1875000000000005e-06, "loss": 0.008, "num_tokens": 52440063.0, "reward": 0.404575914144516, "reward_std": 0.10092879272997379, "rewards/accuracy_reward/mean": 0.08258928614668548, "rewards/accuracy_reward/std": 0.24078447185456753, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3219866082072258, "rewards/tag_count_reward/std": 0.11301804892718792, "step": 100 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 1006.8973693847656, "completions/mean_terminated_length": 884.0690765380859, "completions/min_length": 715.75, "completions/min_terminated_length": 715.75, "epoch": 0.030169516839668432, "grad_norm": 0.0852271243929863, "kl": 0.0099334716796875, "learning_rate": 6.25e-06, "loss": 0.0021, "num_tokens": 52966161.0, "reward": 0.4218750298023224, "reward_std": 0.1894159410148859, "rewards/accuracy_reward/mean": 0.09598214365541935, "rewards/accuracy_reward/std": 0.2415010817348957, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3258928582072258, "rewards/tag_count_reward/std": 0.11025375686585903, "step": 101 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 934.25, "completions/mean_length": 1006.6049499511719, "completions/mean_terminated_length": 826.3046417236328, "completions/min_length": 694.75, "completions/min_terminated_length": 694.75, "epoch": 0.030468224927189903, "grad_norm": 0.09161662310361862, "kl": 0.010162353515625, "learning_rate": 6.3125e-06, "loss": 0.0064, "num_tokens": 53487472.0, "reward": 0.3431919813156128, "reward_std": 0.1002694247290492, "rewards/accuracy_reward/mean": 0.03571428614668548, "rewards/accuracy_reward/std": 0.14183956757187843, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3074776828289032, "rewards/tag_count_reward/std": 0.10469852946698666, "step": 102 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7723214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 953.200927734375, "completions/mean_terminated_length": 750.7011871337891, "completions/min_length": 515.5, "completions/min_terminated_length": 515.5, "epoch": 0.030766933014711374, "grad_norm": 0.10204487293958664, "kl": 0.013336181640625, "learning_rate": 6.375e-06, "loss": 0.0158, "num_tokens": 53981610.0, "reward": 0.4570312723517418, "reward_std": 0.10794909577816725, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.27393461018800735, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3387276753783226, "rewards/tag_count_reward/std": 0.11729110963642597, "step": 103 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.25, "completions/mean_length": 971.1518249511719, "completions/mean_terminated_length": 715.9982681274414, "completions/min_length": 524.25, "completions/min_terminated_length": 524.25, "epoch": 0.03106564110223284, "grad_norm": 0.091836117208004, "kl": 0.0107574462890625, "learning_rate": 6.437500000000001e-06, "loss": 0.0104, "num_tokens": 54488798.0, "reward": 0.3911830484867096, "reward_std": 0.12411053851246834, "rewards/accuracy_reward/mean": 0.06473214505240321, "rewards/accuracy_reward/std": 0.22852186858654022, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3264508917927742, "rewards/tag_count_reward/std": 0.11406475305557251, "step": 104 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9709821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 746.5, "completions/mean_length": 1020.3460235595703, "completions/mean_terminated_length": 682.53125, "completions/min_length": 877.75, "completions/min_terminated_length": 621.75, "epoch": 0.03136434918975431, "grad_norm": 0.09543833881616592, "kl": 0.0090789794921875, "learning_rate": 6.5000000000000004e-06, "loss": 0.0037, "num_tokens": 55026905.0, "reward": 0.3152901902794838, "reward_std": 0.07464526314288378, "rewards/accuracy_reward/mean": 0.03199404804036021, "rewards/accuracy_reward/std": 0.12870225310325623, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2840401753783226, "rewards/tag_count_reward/std": 0.08560692332684994, "step": 105 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 1024.0, "completions/max_terminated_length": 963.75, "completions/mean_length": 965.3504791259766, "completions/mean_terminated_length": 735.4955139160156, "completions/min_length": 480.75, "completions/min_terminated_length": 480.75, "epoch": 0.03166305727727578, "grad_norm": 0.09738217294216156, "kl": 0.0138397216796875, "learning_rate": 6.5625e-06, "loss": 0.019, "num_tokens": 55524774.0, "reward": 0.4603794887661934, "reward_std": 0.1497757826000452, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.34606851637363434, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3219866156578064, "rewards/tag_count_reward/std": 0.11250177398324013, "step": 106 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 967.591552734375, "completions/mean_terminated_length": 711.7642974853516, "completions/min_length": 452.25, "completions/min_terminated_length": 452.25, "epoch": 0.03196176536479725, "grad_norm": 0.09197632223367691, "kl": 0.013397216796875, "learning_rate": 6.625e-06, "loss": 0.003, "num_tokens": 56026719.0, "reward": 0.3744419813156128, "reward_std": 0.10537490248680115, "rewards/accuracy_reward/mean": 0.05133928544819355, "rewards/accuracy_reward/std": 0.166031863540411, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3231026828289032, "rewards/tag_count_reward/std": 0.11190779320895672, "step": 107 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 998.4553985595703, "completions/mean_terminated_length": 825.2649078369141, "completions/min_length": 573.0, "completions/min_terminated_length": 573.0, "epoch": 0.03226047345231872, "grad_norm": 0.10796286910772324, "kl": 0.0130767822265625, "learning_rate": 6.6875e-06, "loss": 0.0111, "num_tokens": 56556875.0, "reward": 0.412946455180645, "reward_std": 0.1736610196530819, "rewards/accuracy_reward/mean": 0.10044642863795161, "rewards/accuracy_reward/std": 0.2849069759249687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3125000074505806, "rewards/tag_count_reward/std": 0.10522296279668808, "step": 108 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8258928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.5, "completions/mean_length": 971.6004791259766, "completions/mean_terminated_length": 767.7700653076172, "completions/min_length": 543.5, "completions/min_terminated_length": 543.5, "epoch": 0.03255918153984019, "grad_norm": 0.0926847830414772, "kl": 0.014312744140625, "learning_rate": 6.750000000000001e-06, "loss": 0.0082, "num_tokens": 57056984.0, "reward": 0.4531250149011612, "reward_std": 0.16078037582337856, "rewards/accuracy_reward/mean": 0.12723214086145163, "rewards/accuracy_reward/std": 0.3103998303413391, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3258928507566452, "rewards/tag_count_reward/std": 0.11160900443792343, "step": 109 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.25, "completions/mean_length": 958.8504791259766, "completions/mean_terminated_length": 748.4986267089844, "completions/min_length": 485.0, "completions/min_terminated_length": 485.0, "epoch": 0.032857889627361664, "grad_norm": 0.08852526545524597, "kl": 0.0176544189453125, "learning_rate": 6.8125e-06, "loss": 0.0048, "num_tokens": 57555861.0, "reward": 0.412388414144516, "reward_std": 0.09210400097072124, "rewards/accuracy_reward/mean": 0.08258928498253226, "rewards/accuracy_reward/std": 0.2024109959602356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3297991082072258, "rewards/tag_count_reward/std": 0.10925194807350636, "step": 110 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 1013.2522735595703, "completions/mean_terminated_length": 878.2148895263672, "completions/min_length": 723.25, "completions/min_terminated_length": 723.25, "epoch": 0.03315659771488313, "grad_norm": 0.08346787840127945, "kl": 0.012725830078125, "learning_rate": 6.875e-06, "loss": 0.0044, "num_tokens": 58086390.0, "reward": 0.3716518059372902, "reward_std": 0.13061585556715727, "rewards/accuracy_reward/mean": 0.05580357043072581, "rewards/accuracy_reward/std": 0.17411095276474953, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3158482164144516, "rewards/tag_count_reward/std": 0.10568141005933285, "step": 111 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.890625, "completions/max_length": 1024.0, "completions/max_terminated_length": 711.5, "completions/mean_length": 987.950927734375, "completions/mean_terminated_length": 497.2418518066406, "completions/min_length": 578.0, "completions/min_terminated_length": 322.0, "epoch": 0.0334553058024046, "grad_norm": 0.09269267320632935, "kl": 0.0143280029296875, "learning_rate": 6.9375e-06, "loss": 0.0075, "num_tokens": 58599568.0, "reward": 0.4095982387661934, "reward_std": 0.11334617622196674, "rewards/accuracy_reward/mean": 0.10267857019789517, "rewards/accuracy_reward/std": 0.2694157436490059, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3069196417927742, "rewards/tag_count_reward/std": 0.09954702854156494, "step": 112 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 1000.716552734375, "completions/mean_terminated_length": 832.7640686035156, "completions/min_length": 638.25, "completions/min_terminated_length": 638.25, "epoch": 0.03375401388992607, "grad_norm": 0.10000244528055191, "kl": 0.014068603515625, "learning_rate": 7e-06, "loss": 0.0123, "num_tokens": 59118513.0, "reward": 0.353236623108387, "reward_std": 0.12049346137791872, "rewards/accuracy_reward/mean": 0.03794642933644354, "rewards/accuracy_reward/std": 0.1517776157706976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3152901753783226, "rewards/tag_count_reward/std": 0.10844751447439194, "step": 113 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 975.8616485595703, "completions/mean_terminated_length": 787.6055145263672, "completions/min_length": 517.75, "completions/min_terminated_length": 517.75, "epoch": 0.03405272197744754, "grad_norm": 0.09538033604621887, "kl": 0.0156707763671875, "learning_rate": 7.062500000000001e-06, "loss": 0.0063, "num_tokens": 59627443.0, "reward": 0.4246651977300644, "reward_std": 0.11325054243206978, "rewards/accuracy_reward/mean": 0.08705356996506453, "rewards/accuracy_reward/std": 0.2268107682466507, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3376116082072258, "rewards/tag_count_reward/std": 0.11833509244024754, "step": 114 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9263392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.25, "completions/mean_length": 1007.638427734375, "completions/mean_terminated_length": 881.984375, "completions/min_length": 749.25, "completions/min_terminated_length": 749.25, "epoch": 0.03435143006496901, "grad_norm": 0.09803593903779984, "kl": 0.0133056640625, "learning_rate": 7.125e-06, "loss": 0.0047, "num_tokens": 60149201.0, "reward": 0.3152901902794838, "reward_std": 0.10193910263478756, "rewards/accuracy_reward/mean": 0.01562500069849193, "rewards/accuracy_reward/std": 0.10348234511911869, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2996651828289032, "rewards/tag_count_reward/std": 0.09711368195712566, "step": 115 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 987.5178985595703, "completions/mean_terminated_length": 841.4068756103516, "completions/min_length": 612.25, "completions/min_terminated_length": 612.25, "epoch": 0.034650138152490476, "grad_norm": 0.10468866676092148, "kl": 0.01495361328125, "learning_rate": 7.1875e-06, "loss": 0.0159, "num_tokens": 60661225.0, "reward": 0.3331473395228386, "reward_std": 0.06036594044417143, "rewards/accuracy_reward/mean": 0.0022321429569274187, "rewards/accuracy_reward/std": 0.023622779175639153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3309151753783226, "rewards/tag_count_reward/std": 0.10821034759283066, "step": 116 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 1005.3303985595703, "completions/mean_terminated_length": 901.4839324951172, "completions/min_length": 742.25, "completions/min_terminated_length": 742.25, "epoch": 0.034948846240011947, "grad_norm": 0.08738310635089874, "kl": 0.0131683349609375, "learning_rate": 7.25e-06, "loss": 0.0036, "num_tokens": 61184269.0, "reward": 0.3895089402794838, "reward_std": 0.11042762082070112, "rewards/accuracy_reward/mean": 0.07589285727590322, "rewards/accuracy_reward/std": 0.22038429975509644, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3136160746216774, "rewards/tag_count_reward/std": 0.10071817971765995, "step": 117 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.25, "completions/mean_length": 950.7411193847656, "completions/mean_terminated_length": 750.1588821411133, "completions/min_length": 503.75, "completions/min_terminated_length": 503.75, "epoch": 0.03524755432753342, "grad_norm": 0.09623870998620987, "kl": 0.0157318115234375, "learning_rate": 7.3125e-06, "loss": 0.0085, "num_tokens": 61681193.0, "reward": 0.4017857313156128, "reward_std": 0.11387522565200925, "rewards/accuracy_reward/mean": 0.07589285634458065, "rewards/accuracy_reward/std": 0.224321898072958, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3281250074505806, "rewards/tag_count_reward/std": 0.11350252851843834, "step": 118 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8125000000000001, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 991.4018402099609, "completions/mean_terminated_length": 854.3889312744141, "completions/min_length": 646.25, "completions/min_terminated_length": 646.25, "epoch": 0.03554626241505489, "grad_norm": 0.10617967694997787, "kl": 0.01641845703125, "learning_rate": 7.375000000000001e-06, "loss": 0.0148, "num_tokens": 62203053.0, "reward": 0.5758928880095482, "reward_std": 0.22641940787434578, "rewards/accuracy_reward/mean": 0.20535713993012905, "rewards/accuracy_reward/std": 0.3776763379573822, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3705357164144516, "rewards/tag_count_reward/std": 0.1252911426126957, "step": 119 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 705.25, "completions/mean_length": 1002.3080749511719, "completions/mean_terminated_length": 609.3678283691406, "completions/min_length": 764.5, "completions/min_terminated_length": 508.5, "epoch": 0.03584497050257636, "grad_norm": 0.06674060225486755, "kl": 0.011932373046875, "learning_rate": 7.437500000000001e-06, "loss": 0.002, "num_tokens": 62725799.0, "reward": 0.4090402126312256, "reward_std": 0.05695156310684979, "rewards/accuracy_reward/mean": 0.10937500186264515, "rewards/accuracy_reward/std": 0.26449109613895416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2996651753783226, "rewards/tag_count_reward/std": 0.08640421135351062, "step": 120 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7991071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 982.6696929931641, "completions/mean_terminated_length": 834.9366607666016, "completions/min_length": 593.75, "completions/min_terminated_length": 593.75, "epoch": 0.03614367859009783, "grad_norm": 0.08994851261377335, "kl": 0.0143890380859375, "learning_rate": 7.500000000000001e-06, "loss": 0.0021, "num_tokens": 63244563.0, "reward": 0.534598246216774, "reward_std": 0.11422513937577605, "rewards/accuracy_reward/mean": 0.1919642835855484, "rewards/accuracy_reward/std": 0.3891778513789177, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3426339328289032, "rewards/tag_count_reward/std": 0.11720650270581245, "step": 121 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.90625, "completions/max_length": 1024.0, "completions/max_terminated_length": 754.5, "completions/mean_length": 992.1361846923828, "completions/mean_terminated_length": 535.5632705688477, "completions/min_length": 651.25, "completions/min_terminated_length": 395.25, "epoch": 0.036442386677619294, "grad_norm": 0.09424477070569992, "kl": 0.0137481689453125, "learning_rate": 7.5625e-06, "loss": 0.0055, "num_tokens": 63763248.0, "reward": 0.415178582072258, "reward_std": 0.12456307606771588, "rewards/accuracy_reward/mean": 0.08482143143191934, "rewards/accuracy_reward/std": 0.21731038764119148, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3303571417927742, "rewards/tag_count_reward/std": 0.10392123088240623, "step": 122 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 996.2344207763672, "completions/mean_terminated_length": 846.7076873779297, "completions/min_length": 670.25, "completions/min_terminated_length": 670.25, "epoch": 0.036741094765140765, "grad_norm": 0.10092459619045258, "kl": 0.013458251953125, "learning_rate": 7.625e-06, "loss": 0.0035, "num_tokens": 64282345.0, "reward": 0.420200914144516, "reward_std": 0.09516235906630754, "rewards/accuracy_reward/mean": 0.08705357206054032, "rewards/accuracy_reward/std": 0.21059381030499935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3331473171710968, "rewards/tag_count_reward/std": 0.10969305969774723, "step": 123 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 982.0804138183594, "completions/mean_terminated_length": 832.0240325927734, "completions/min_length": 576.25, "completions/min_terminated_length": 576.25, "epoch": 0.037039802852662236, "grad_norm": 0.11166391521692276, "kl": 0.0167694091796875, "learning_rate": 7.6875e-06, "loss": 0.0005, "num_tokens": 64795357.0, "reward": 0.5457589626312256, "reward_std": 0.17364349029958248, "rewards/accuracy_reward/mean": 0.20312499441206455, "rewards/accuracy_reward/std": 0.3942814990878105, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3426339253783226, "rewards/tag_count_reward/std": 0.11521797068417072, "step": 124 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 991.5670166015625, "completions/mean_terminated_length": 842.0220794677734, "completions/min_length": 670.5, "completions/min_terminated_length": 670.5, "epoch": 0.03733851094018371, "grad_norm": 0.09037909656763077, "kl": 0.014129638671875, "learning_rate": 7.75e-06, "loss": 0.0064, "num_tokens": 65306379.0, "reward": 0.404017873108387, "reward_std": 0.06245558522641659, "rewards/accuracy_reward/mean": 0.0691964291036129, "rewards/accuracy_reward/std": 0.17340151220560074, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3348214253783226, "rewards/tag_count_reward/std": 0.1170163806527853, "step": 125 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.25, "completions/mean_length": 1002.154052734375, "completions/mean_terminated_length": 856.4403076171875, "completions/min_length": 649.25, "completions/min_terminated_length": 649.25, "epoch": 0.03763721902770518, "grad_norm": 0.09394252300262451, "kl": 0.0139617919921875, "learning_rate": 7.8125e-06, "loss": 0.0025, "num_tokens": 65830112.0, "reward": 0.3900669738650322, "reward_std": 0.13783142808824778, "rewards/accuracy_reward/mean": 0.08035714644938707, "rewards/accuracy_reward/std": 0.22235263884067535, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3097098171710968, "rewards/tag_count_reward/std": 0.10563334822654724, "step": 126 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.0, "completions/mean_length": 994.5380096435547, "completions/mean_terminated_length": 855.5021514892578, "completions/min_length": 697.5, "completions/min_terminated_length": 697.5, "epoch": 0.03793592711522664, "grad_norm": 0.0996580570936203, "kl": 0.015350341796875, "learning_rate": 7.875e-06, "loss": 0.0052, "num_tokens": 66347601.0, "reward": 0.467633955180645, "reward_std": 0.1911172904074192, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.33153095096349716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3404017835855484, "rewards/tag_count_reward/std": 0.11741730570793152, "step": 127 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 1007.7455749511719, "completions/mean_terminated_length": 905.5273895263672, "completions/min_length": 713.75, "completions/min_terminated_length": 713.75, "epoch": 0.03823463520274811, "grad_norm": 0.10126273334026337, "kl": 0.0154571533203125, "learning_rate": 7.9375e-06, "loss": 0.0079, "num_tokens": 66867471.0, "reward": 0.4335937723517418, "reward_std": 0.12453044205904007, "rewards/accuracy_reward/mean": 0.10044642654247582, "rewards/accuracy_reward/std": 0.2777272127568722, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3331473171710968, "rewards/tag_count_reward/std": 0.10574928857386112, "step": 128 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 1024.0, "completions/max_terminated_length": 648.5, "completions/mean_length": 958.2232513427734, "completions/mean_terminated_length": 497.5027770996094, "completions/min_length": 605.0, "completions/min_terminated_length": 349.0, "epoch": 0.038533343290269584, "grad_norm": 0.092801533639431, "kl": 0.015472412109375, "learning_rate": 8.000000000000001e-06, "loss": 0.0074, "num_tokens": 67366963.0, "reward": 0.4375000298023224, "reward_std": 0.11884741112589836, "rewards/accuracy_reward/mean": 0.10788690391927958, "rewards/accuracy_reward/std": 0.27257027104496956, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3303571492433548, "rewards/tag_count_reward/std": 0.1000912906602025, "step": 129 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7857142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.0, "completions/mean_length": 972.0893249511719, "completions/mean_terminated_length": 783.8434906005859, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.038832051377791055, "grad_norm": 0.10908523201942444, "kl": 0.0164642333984375, "learning_rate": 8.062500000000001e-06, "loss": 0.0115, "num_tokens": 67870075.0, "reward": 0.5251116380095482, "reward_std": 0.1369409989565611, "rewards/accuracy_reward/mean": 0.1674107164144516, "rewards/accuracy_reward/std": 0.37056972086429596, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3577008917927742, "rewards/tag_count_reward/std": 0.1229797936975956, "step": 130 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9464285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 694.5, "completions/mean_length": 1007.7500457763672, "completions/mean_terminated_length": 593.7698059082031, "completions/min_length": 749.75, "completions/min_terminated_length": 493.75, "epoch": 0.039130759465312526, "grad_norm": 0.10558289289474487, "kl": 0.015869140625, "learning_rate": 8.125000000000001e-06, "loss": 0.0059, "num_tokens": 68390715.0, "reward": 0.436383955180645, "reward_std": 0.19402697309851646, "rewards/accuracy_reward/mean": 0.12276785681024194, "rewards/accuracy_reward/std": 0.302160058170557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3136160671710968, "rewards/tag_count_reward/std": 0.10821128822863102, "step": 131 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8169642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.75, "completions/mean_length": 972.5313110351562, "completions/mean_terminated_length": 731.2725830078125, "completions/min_length": 439.5, "completions/min_terminated_length": 439.5, "epoch": 0.03942946755283399, "grad_norm": 0.13657857477664948, "kl": 0.017181396484375, "learning_rate": 8.1875e-06, "loss": 0.0357, "num_tokens": 68895161.0, "reward": 0.3577009066939354, "reward_std": 0.08253514114767313, "rewards/accuracy_reward/mean": 0.01785714295692742, "rewards/accuracy_reward/std": 0.08441012538969517, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.33984375, "rewards/tag_count_reward/std": 0.11744294129312038, "step": 132 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7924107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.75, "completions/mean_length": 962.9085235595703, "completions/mean_terminated_length": 744.7494506835938, "completions/min_length": 473.75, "completions/min_terminated_length": 473.75, "epoch": 0.03972817564035546, "grad_norm": 0.17997898161411285, "kl": 0.023590087890625, "learning_rate": 8.25e-06, "loss": 0.0314, "num_tokens": 69395536.0, "reward": 0.4854911044239998, "reward_std": 0.1576190385967493, "rewards/accuracy_reward/mean": 0.14955357182770967, "rewards/accuracy_reward/std": 0.33755991607904434, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3359375074505806, "rewards/tag_count_reward/std": 0.11410903558135033, "step": 133 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 949.1495971679688, "completions/mean_terminated_length": 726.7075347900391, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.04002688372787693, "grad_norm": 0.11717874556779861, "kl": 0.0208740234375, "learning_rate": 8.3125e-06, "loss": 0.0179, "num_tokens": 69891139.0, "reward": 0.3872767984867096, "reward_std": 0.14705059863626957, "rewards/accuracy_reward/mean": 0.03348214388824999, "rewards/accuracy_reward/std": 0.16938812844455242, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3537946492433548, "rewards/tag_count_reward/std": 0.12109305523335934, "step": 134 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7834821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 967.6585388183594, "completions/mean_terminated_length": 786.7568969726562, "completions/min_length": 581.5, "completions/min_terminated_length": 581.5, "epoch": 0.0403255918153984, "grad_norm": 0.10820689797401428, "kl": 0.023284912109375, "learning_rate": 8.375e-06, "loss": 0.0146, "num_tokens": 70399354.0, "reward": 0.4866071715950966, "reward_std": 0.14142471924424171, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.3225584290921688, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.359375, "rewards/tag_count_reward/std": 0.12299641221761703, "step": 135 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 916.5, "completions/mean_length": 1007.0491485595703, "completions/mean_terminated_length": 798.364990234375, "completions/min_length": 659.75, "completions/min_terminated_length": 659.75, "epoch": 0.04062429990291987, "grad_norm": 0.09435324370861053, "kl": 0.0179290771484375, "learning_rate": 8.4375e-06, "loss": 0.0151, "num_tokens": 70929856.0, "reward": 0.3716517984867096, "reward_std": 0.10880368575453758, "rewards/accuracy_reward/mean": 0.07142856996506453, "rewards/accuracy_reward/std": 0.2125668302178383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.300223208963871, "rewards/tag_count_reward/std": 0.09540112502872944, "step": 136 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 995.1808471679688, "completions/mean_terminated_length": 859.9916839599609, "completions/min_length": 673.5, "completions/min_terminated_length": 673.5, "epoch": 0.040923007990441344, "grad_norm": 0.10471641272306442, "kl": 0.022857666015625, "learning_rate": 8.5e-06, "loss": 0.0169, "num_tokens": 71441169.0, "reward": 0.6121652126312256, "reward_std": 0.24083196744322777, "rewards/accuracy_reward/mean": 0.2522321417927742, "rewards/accuracy_reward/std": 0.4269026294350624, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3599330335855484, "rewards/tag_count_reward/std": 0.12167127802968025, "step": 137 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.78125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 983.6473541259766, "completions/mean_terminated_length": 842.4509735107422, "completions/min_length": 627.25, "completions/min_terminated_length": 627.25, "epoch": 0.04122171607796281, "grad_norm": 0.09313499182462692, "kl": 0.023651123046875, "learning_rate": 8.5625e-06, "loss": 0.0084, "num_tokens": 71953827.0, "reward": 0.4068080484867096, "reward_std": 0.09883693978190422, "rewards/accuracy_reward/mean": 0.06845238292589784, "rewards/accuracy_reward/std": 0.23942669108510017, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.33984375, "rewards/tag_count_reward/std": 0.11783022992312908, "step": 138 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7566964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.25, "completions/mean_length": 963.9866485595703, "completions/mean_terminated_length": 772.6888122558594, "completions/min_length": 541.0, "completions/min_terminated_length": 541.0, "epoch": 0.04152042416548428, "grad_norm": 0.11240748316049576, "kl": 0.024688720703125, "learning_rate": 8.625000000000001e-06, "loss": 0.0138, "num_tokens": 72455853.0, "reward": 0.4827009215950966, "reward_std": 0.14099881052970886, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.2881506457924843, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3465401828289032, "rewards/tag_count_reward/std": 0.11782664805650711, "step": 139 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7790178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 962.5647735595703, "completions/mean_terminated_length": 779.9755859375, "completions/min_length": 497.25, "completions/min_terminated_length": 497.25, "epoch": 0.04181913225300575, "grad_norm": 0.09669417887926102, "kl": 0.025390625, "learning_rate": 8.687500000000001e-06, "loss": 0.0107, "num_tokens": 72960170.0, "reward": 0.5318080633878708, "reward_std": 0.16306851245462894, "rewards/accuracy_reward/mean": 0.17633928474970162, "rewards/accuracy_reward/std": 0.33461966924369335, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3554687425494194, "rewards/tag_count_reward/std": 0.12261403352022171, "step": 140 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.5, "completions/mean_length": 993.9821929931641, "completions/mean_terminated_length": 836.2030029296875, "completions/min_length": 610.5, "completions/min_terminated_length": 610.5, "epoch": 0.04211784034052722, "grad_norm": 0.1000056341290474, "kl": 0.024658203125, "learning_rate": 8.750000000000001e-06, "loss": 0.0046, "num_tokens": 73475762.0, "reward": 0.5239955484867096, "reward_std": 0.17427565529942513, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.36814387887716293, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.34765625, "rewards/tag_count_reward/std": 0.1221070159226656, "step": 141 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.671875, "completions/max_length": 1024.0, "completions/max_terminated_length": 990.75, "completions/mean_length": 951.4821929931641, "completions/mean_terminated_length": 804.8074951171875, "completions/min_length": 494.0, "completions/min_terminated_length": 494.0, "epoch": 0.04241654842804869, "grad_norm": 0.10881073027849197, "kl": 0.02783203125, "learning_rate": 8.8125e-06, "loss": 0.0125, "num_tokens": 73974106.0, "reward": 0.5212053954601288, "reward_std": 0.13922698982059956, "rewards/accuracy_reward/mean": 0.1339285713620484, "rewards/accuracy_reward/std": 0.29453569650650024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.387276791036129, "rewards/tag_count_reward/std": 0.11874076165258884, "step": 142 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.75, "completions/mean_length": 990.3928985595703, "completions/mean_terminated_length": 786.4078521728516, "completions/min_length": 620.5, "completions/min_terminated_length": 620.5, "epoch": 0.042715256515570156, "grad_norm": 0.10796710103750229, "kl": 0.02532958984375, "learning_rate": 8.875e-06, "loss": 0.0072, "num_tokens": 74497194.0, "reward": 0.4531250149011612, "reward_std": 0.17116067791357636, "rewards/accuracy_reward/mean": 0.11607143096625805, "rewards/accuracy_reward/std": 0.25644827634096146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3370535671710968, "rewards/tag_count_reward/std": 0.11572289653122425, "step": 143 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.5, "completions/mean_length": 968.1674499511719, "completions/mean_terminated_length": 750.0188140869141, "completions/min_length": 521.0, "completions/min_terminated_length": 521.0, "epoch": 0.04301396460309163, "grad_norm": 0.09637793898582458, "kl": 0.025146484375, "learning_rate": 8.9375e-06, "loss": 0.0134, "num_tokens": 75006885.0, "reward": 0.435825914144516, "reward_std": 0.10753383114933968, "rewards/accuracy_reward/mean": 0.0982142873108387, "rewards/accuracy_reward/std": 0.197520412504673, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3376116007566452, "rewards/tag_count_reward/std": 0.11608073860406876, "step": 144 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7834821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 984.5982513427734, "completions/mean_terminated_length": 847.9643707275391, "completions/min_length": 582.0, "completions/min_terminated_length": 582.0, "epoch": 0.0433126726906131, "grad_norm": 0.116435207426548, "kl": 0.027679443359375, "learning_rate": 9e-06, "loss": 0.008, "num_tokens": 75513841.0, "reward": 0.5708705559372902, "reward_std": 0.17098627798259258, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3156479597091675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3833705335855484, "rewards/tag_count_reward/std": 0.11773349717259407, "step": 145 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.25, "completions/mean_length": 993.7701263427734, "completions/mean_terminated_length": 855.615966796875, "completions/min_length": 622.5, "completions/min_terminated_length": 622.5, "epoch": 0.04361138077813457, "grad_norm": 0.09868639707565308, "kl": 0.02490234375, "learning_rate": 9.0625e-06, "loss": 0.0153, "num_tokens": 76037434.0, "reward": 0.3655134066939354, "reward_std": 0.10221273265779018, "rewards/accuracy_reward/mean": 0.022321428637951612, "rewards/accuracy_reward/std": 0.10133291408419609, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3431919738650322, "rewards/tag_count_reward/std": 0.11949489079415798, "step": 146 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8437500000000001, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.75, "completions/mean_length": 996.2299652099609, "completions/mean_terminated_length": 845.2417755126953, "completions/min_length": 637.5, "completions/min_terminated_length": 637.5, "epoch": 0.04391008886565604, "grad_norm": 0.11356164515018463, "kl": 0.024566650390625, "learning_rate": 9.125e-06, "loss": 0.0086, "num_tokens": 76556289.0, "reward": 0.4921875149011612, "reward_std": 0.15090367011725903, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.362604022026062, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3359375, "rewards/tag_count_reward/std": 0.11712843738496304, "step": 147 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 989.0692443847656, "completions/mean_terminated_length": 839.6440887451172, "completions/min_length": 632.0, "completions/min_terminated_length": 632.0, "epoch": 0.04420879695317751, "grad_norm": 0.10536891222000122, "kl": 0.0225830078125, "learning_rate": 9.1875e-06, "loss": 0.0059, "num_tokens": 77076336.0, "reward": 0.4196428805589676, "reward_std": 0.16018225252628326, "rewards/accuracy_reward/mean": 0.08705357159487903, "rewards/accuracy_reward/std": 0.2452363185584545, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3325892835855484, "rewards/tag_count_reward/std": 0.11590796336531639, "step": 148 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 966.4911193847656, "completions/mean_terminated_length": 772.5183563232422, "completions/min_length": 520.25, "completions/min_terminated_length": 520.25, "epoch": 0.044507505040698975, "grad_norm": 0.09366322308778763, "kl": 0.024505615234375, "learning_rate": 9.250000000000001e-06, "loss": 0.0107, "num_tokens": 77580300.0, "reward": 0.4335937649011612, "reward_std": 0.14456819230690598, "rewards/accuracy_reward/mean": 0.08482143096625805, "rewards/accuracy_reward/std": 0.2371555119752884, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3487723246216774, "rewards/tag_count_reward/std": 0.1210821308195591, "step": 149 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 976.75, "completions/mean_length": 1009.4330902099609, "completions/mean_terminated_length": 876.7208251953125, "completions/min_length": 760.5, "completions/min_terminated_length": 760.5, "epoch": 0.044806213128220446, "grad_norm": 0.1058359369635582, "kl": 0.02197265625, "learning_rate": 9.312500000000001e-06, "loss": 0.0032, "num_tokens": 78113182.0, "reward": 0.372209832072258, "reward_std": 0.10542005114257336, "rewards/accuracy_reward/mean": 0.05580357322469354, "rewards/accuracy_reward/std": 0.14869003370404243, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.31640625, "rewards/tag_count_reward/std": 0.10919207893311977, "step": 150 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8638392857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 941.25, "completions/mean_length": 986.1495971679688, "completions/mean_terminated_length": 755.9327545166016, "completions/min_length": 570.0, "completions/min_terminated_length": 570.0, "epoch": 0.04510492121574192, "grad_norm": 0.1069486141204834, "kl": 0.0205078125, "learning_rate": 9.375000000000001e-06, "loss": 0.0119, "num_tokens": 78631825.0, "reward": 0.4375000149011612, "reward_std": 0.1366596668958664, "rewards/accuracy_reward/mean": 0.12946428661234677, "rewards/accuracy_reward/std": 0.30250896885991096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3080357164144516, "rewards/tag_count_reward/std": 0.1012576874345541, "step": 151 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.921875, "completions/max_length": 1024.0, "completions/max_terminated_length": 854.75, "completions/mean_length": 996.0580749511719, "completions/mean_terminated_length": 751.2544708251953, "completions/min_length": 645.5, "completions/min_terminated_length": 645.5, "epoch": 0.04540362930326339, "grad_norm": 0.10738148540258408, "kl": 0.02001953125, "learning_rate": 9.4375e-06, "loss": 0.0038, "num_tokens": 79148971.0, "reward": 0.384486623108387, "reward_std": 0.1313024628907442, "rewards/accuracy_reward/mean": 0.07142857322469354, "rewards/accuracy_reward/std": 0.20418310537934303, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3130580335855484, "rewards/tag_count_reward/std": 0.10441209189593792, "step": 152 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8147321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 992.0223541259766, "completions/mean_terminated_length": 842.3602447509766, "completions/min_length": 643.5, "completions/min_terminated_length": 643.5, "epoch": 0.04570233739078486, "grad_norm": 0.10521404445171356, "kl": 0.02276611328125, "learning_rate": 9.5e-06, "loss": 0.0098, "num_tokens": 79662565.0, "reward": 0.459263414144516, "reward_std": 0.14735347963869572, "rewards/accuracy_reward/mean": 0.11383928474970162, "rewards/accuracy_reward/std": 0.20654179342091084, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3454241082072258, "rewards/tag_count_reward/std": 0.11497914791107178, "step": 153 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7767857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 979.7098693847656, "completions/mean_terminated_length": 839.7511444091797, "completions/min_length": 642.75, "completions/min_terminated_length": 642.75, "epoch": 0.04600104547830632, "grad_norm": 0.09784567356109619, "kl": 0.025970458984375, "learning_rate": 9.562500000000002e-06, "loss": 0.0077, "num_tokens": 80171907.0, "reward": 0.5518973469734192, "reward_std": 0.1508705373853445, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.36578499153256416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.35546875, "rewards/tag_count_reward/std": 0.12110238149762154, "step": 154 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 989.2098541259766, "completions/mean_terminated_length": 827.1639862060547, "completions/min_length": 637.5, "completions/min_terminated_length": 637.5, "epoch": 0.04629975356582779, "grad_norm": 0.09995438158512115, "kl": 0.02203369140625, "learning_rate": 9.625e-06, "loss": 0.0042, "num_tokens": 80686817.0, "reward": 0.4330357313156128, "reward_std": 0.11732741491869092, "rewards/accuracy_reward/mean": 0.10565476352348924, "rewards/accuracy_reward/std": 0.2304641343653202, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3370535671710968, "rewards/tag_count_reward/std": 0.11890756338834763, "step": 155 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.953125, "completions/max_length": 1024.0, "completions/max_terminated_length": 715.25, "completions/mean_length": 1016.0937805175781, "completions/mean_terminated_length": 626.1309661865234, "completions/min_length": 783.25, "completions/min_terminated_length": 527.25, "epoch": 0.046598461653349264, "grad_norm": 0.10353723913431168, "kl": 0.021514892578125, "learning_rate": 9.6875e-06, "loss": 0.0053, "num_tokens": 81218523.0, "reward": 0.3415178656578064, "reward_std": 0.14107644092291594, "rewards/accuracy_reward/mean": 0.04910714249126613, "rewards/accuracy_reward/std": 0.15639781020581722, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.2924107164144516, "rewards/tag_count_reward/std": 0.09083993546664715, "step": 156 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7991071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 967.9486999511719, "completions/mean_terminated_length": 757.6594543457031, "completions/min_length": 571.0, "completions/min_terminated_length": 571.0, "epoch": 0.046897169740870735, "grad_norm": 0.08798649162054062, "kl": 0.023773193359375, "learning_rate": 9.75e-06, "loss": 0.0061, "num_tokens": 81731460.0, "reward": 0.4988839477300644, "reward_std": 0.09683948196470737, "rewards/accuracy_reward/mean": 0.16741071501746774, "rewards/accuracy_reward/std": 0.3368151970207691, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.331473208963871, "rewards/tag_count_reward/std": 0.10999250784516335, "step": 157 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 1019.1875457763672, "completions/mean_terminated_length": 942.9750061035156, "completions/min_length": 823.75, "completions/min_terminated_length": 823.75, "epoch": 0.047195877828392206, "grad_norm": 0.09566624462604523, "kl": 0.02197265625, "learning_rate": 9.8125e-06, "loss": 0.004, "num_tokens": 82254376.0, "reward": 0.361607164144516, "reward_std": 0.12239013984799385, "rewards/accuracy_reward/mean": 0.03348214318975806, "rewards/accuracy_reward/std": 0.14812633767724037, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.328125, "rewards/tag_count_reward/std": 0.11593122780323029, "step": 158 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.5, "completions/mean_length": 977.6719207763672, "completions/mean_terminated_length": 803.5315704345703, "completions/min_length": 630.0, "completions/min_terminated_length": 630.0, "epoch": 0.04749458591591367, "grad_norm": 0.09859690815210342, "kl": 0.0220947265625, "learning_rate": 9.875000000000001e-06, "loss": 0.0057, "num_tokens": 82767509.0, "reward": 0.387834832072258, "reward_std": 0.11089062364771962, "rewards/accuracy_reward/mean": 0.07142856996506453, "rewards/accuracy_reward/std": 0.1625698059797287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.31640625, "rewards/tag_count_reward/std": 0.10586614534258842, "step": 159 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.25, "completions/mean_length": 976.7857666015625, "completions/mean_terminated_length": 798.4152984619141, "completions/min_length": 569.75, "completions/min_terminated_length": 569.75, "epoch": 0.04779329400343514, "grad_norm": 0.11110155284404755, "kl": 0.027008056640625, "learning_rate": 9.937500000000001e-06, "loss": 0.0129, "num_tokens": 83273877.0, "reward": 0.4860491380095482, "reward_std": 0.14514141343533993, "rewards/accuracy_reward/mean": 0.14136905036866665, "rewards/accuracy_reward/std": 0.2902888245880604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3565848246216774, "rewards/tag_count_reward/std": 0.1225469671189785, "step": 160 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7946428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 982.7589721679688, "completions/mean_terminated_length": 828.6819763183594, "completions/min_length": 524.5, "completions/min_terminated_length": 524.5, "epoch": 0.04809200209095661, "grad_norm": 0.09783132374286652, "kl": 0.02276611328125, "learning_rate": 1e-05, "loss": 0.0176, "num_tokens": 83789737.0, "reward": 0.4687500223517418, "reward_std": 0.1299782758578658, "rewards/accuracy_reward/mean": 0.1339285746216774, "rewards/accuracy_reward/std": 0.28448961675167084, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3348214253783226, "rewards/tag_count_reward/std": 0.11835530772805214, "step": 161 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 982.6205902099609, "completions/mean_terminated_length": 822.3916168212891, "completions/min_length": 591.25, "completions/min_terminated_length": 591.25, "epoch": 0.04839071017847808, "grad_norm": 0.10891039669513702, "kl": 0.0269775390625, "learning_rate": 1.0062500000000002e-05, "loss": 0.0073, "num_tokens": 84300335.0, "reward": 0.4302455559372902, "reward_std": 0.1888922154903412, "rewards/accuracy_reward/mean": 0.08482142933644354, "rewards/accuracy_reward/std": 0.23946781270205975, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3454241007566452, "rewards/tag_count_reward/std": 0.11166626214981079, "step": 162 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 1005.8236846923828, "completions/mean_terminated_length": 873.5808868408203, "completions/min_length": 665.0, "completions/min_terminated_length": 665.0, "epoch": 0.048689418265999554, "grad_norm": 0.10104668885469437, "kl": 0.02667236328125, "learning_rate": 1.0125e-05, "loss": 0.0089, "num_tokens": 84824576.0, "reward": 0.439174123108387, "reward_std": 0.12192929349839687, "rewards/accuracy_reward/mean": 0.0848214291036129, "rewards/accuracy_reward/std": 0.2736717537045479, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3543526828289032, "rewards/tag_count_reward/std": 0.1211286298930645, "step": 163 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8258928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 990.9643249511719, "completions/mean_terminated_length": 869.7176818847656, "completions/min_length": 645.5, "completions/min_terminated_length": 645.5, "epoch": 0.048988126353521025, "grad_norm": 0.1138518676161766, "kl": 0.028228759765625, "learning_rate": 1.0187500000000002e-05, "loss": 0.0083, "num_tokens": 85341312.0, "reward": 0.4486607387661934, "reward_std": 0.15054657123982906, "rewards/accuracy_reward/mean": 0.09821428544819355, "rewards/accuracy_reward/std": 0.23948106914758682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3504464328289032, "rewards/tag_count_reward/std": 0.12042740359902382, "step": 164 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 970.2879791259766, "completions/mean_terminated_length": 793.9231872558594, "completions/min_length": 594.5, "completions/min_terminated_length": 594.5, "epoch": 0.04928683444104249, "grad_norm": 0.10322508960962296, "kl": 0.02911376953125, "learning_rate": 1.025e-05, "loss": 0.0085, "num_tokens": 85851041.0, "reward": 0.490513414144516, "reward_std": 0.1791248470544815, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.3273870162665844, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3454241082072258, "rewards/tag_count_reward/std": 0.11846562661230564, "step": 165 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 929.6763763427734, "completions/mean_terminated_length": 795.9193572998047, "completions/min_length": 572.5, "completions/min_terminated_length": 572.5, "epoch": 0.04958554252856396, "grad_norm": 0.10446181893348694, "kl": 0.0338134765625, "learning_rate": 1.0312500000000002e-05, "loss": 0.0061, "num_tokens": 86333040.0, "reward": 0.6618303880095482, "reward_std": 0.1350866798311472, "rewards/accuracy_reward/mean": 0.25892856530845165, "rewards/accuracy_reward/std": 0.3307389169931412, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4029017835855484, "rewards/tag_count_reward/std": 0.10837289690971375, "step": 166 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7388392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 962.9821929931641, "completions/mean_terminated_length": 787.0483856201172, "completions/min_length": 549.0, "completions/min_terminated_length": 549.0, "epoch": 0.04988425061608543, "grad_norm": 0.10311666876077652, "kl": 0.029876708984375, "learning_rate": 1.0375000000000001e-05, "loss": 0.0076, "num_tokens": 86834904.0, "reward": 0.5418526977300644, "reward_std": 0.17456307262182236, "rewards/accuracy_reward/mean": 0.17001487966626883, "rewards/accuracy_reward/std": 0.3585646227002144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3766741156578064, "rewards/tag_count_reward/std": 0.12229110114276409, "step": 167 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.828125, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.0, "completions/mean_length": 987.466552734375, "completions/mean_terminated_length": 856.7914428710938, "completions/min_length": 663.25, "completions/min_terminated_length": 663.25, "epoch": 0.0501829587036069, "grad_norm": 0.11038767546415329, "kl": 0.027252197265625, "learning_rate": 1.04375e-05, "loss": 0.0126, "num_tokens": 87350265.0, "reward": 0.474888414144516, "reward_std": 0.15402423404157162, "rewards/accuracy_reward/mean": 0.11607143050059676, "rewards/accuracy_reward/std": 0.29766281694173813, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3588169664144516, "rewards/tag_count_reward/std": 0.11701750941574574, "step": 168 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7633928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 962.4040679931641, "completions/mean_terminated_length": 795.5112915039062, "completions/min_length": 574.0, "completions/min_terminated_length": 574.0, "epoch": 0.05048166679112837, "grad_norm": 0.10738112777471542, "kl": 0.03057861328125, "learning_rate": 1.0500000000000001e-05, "loss": 0.0182, "num_tokens": 87855102.0, "reward": 0.4458705559372902, "reward_std": 0.123918941244483, "rewards/accuracy_reward/mean": 0.0803571417927742, "rewards/accuracy_reward/std": 0.1707911156117916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3655133917927742, "rewards/tag_count_reward/std": 0.12064259685575962, "step": 169 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9263392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 1012.7098541259766, "completions/mean_terminated_length": 887.8210296630859, "completions/min_length": 674.0, "completions/min_terminated_length": 674.0, "epoch": 0.05078037487864984, "grad_norm": 0.10871543735265732, "kl": 0.0296630859375, "learning_rate": 1.05625e-05, "loss": 0.0158, "num_tokens": 88379708.0, "reward": 0.4324776977300644, "reward_std": 0.16655515506863594, "rewards/accuracy_reward/mean": 0.10044642933644354, "rewards/accuracy_reward/std": 0.25807703845202923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3320312574505806, "rewards/tag_count_reward/std": 0.11658171564340591, "step": 170 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 980.25, "completions/mean_length": 952.8728179931641, "completions/mean_terminated_length": 727.2805480957031, "completions/min_length": 459.25, "completions/min_terminated_length": 459.25, "epoch": 0.05107908296617131, "grad_norm": 0.10347330570220947, "kl": 0.02813720703125, "learning_rate": 1.0625e-05, "loss": 0.008, "num_tokens": 88878291.0, "reward": 0.4469866305589676, "reward_std": 0.12674729898571968, "rewards/accuracy_reward/mean": 0.10267857136204839, "rewards/accuracy_reward/std": 0.2846083976328373, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3443080261349678, "rewards/tag_count_reward/std": 0.11899534799158573, "step": 171 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 984.5, "completions/mean_length": 970.7924499511719, "completions/mean_terminated_length": 764.3941802978516, "completions/min_length": 564.25, "completions/min_terminated_length": 564.25, "epoch": 0.05137779105369278, "grad_norm": 0.10454681515693665, "kl": 0.033233642578125, "learning_rate": 1.0687500000000002e-05, "loss": 0.0051, "num_tokens": 89388198.0, "reward": 0.4737723469734192, "reward_std": 0.1734793223440647, "rewards/accuracy_reward/mean": 0.11830357043072581, "rewards/accuracy_reward/std": 0.29912005737423897, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3554687425494194, "rewards/tag_count_reward/std": 0.12350324913859367, "step": 172 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.75, "completions/mean_length": 976.4933624267578, "completions/mean_terminated_length": 789.9479370117188, "completions/min_length": 591.75, "completions/min_terminated_length": 591.75, "epoch": 0.05167649914121425, "grad_norm": 0.10318484902381897, "kl": 0.03143310546875, "learning_rate": 1.075e-05, "loss": 0.0065, "num_tokens": 89904979.0, "reward": 0.4402901977300644, "reward_std": 0.16092170402407646, "rewards/accuracy_reward/mean": 0.08705357206054032, "rewards/accuracy_reward/std": 0.20971660874783993, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3532366082072258, "rewards/tag_count_reward/std": 0.12010281160473824, "step": 173 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 988.4375457763672, "completions/mean_terminated_length": 865.4902954101562, "completions/min_length": 570.5, "completions/min_terminated_length": 570.5, "epoch": 0.05197520722873572, "grad_norm": 0.1153842881321907, "kl": 0.033294677734375, "learning_rate": 1.0812500000000002e-05, "loss": 0.012, "num_tokens": 90422087.0, "reward": 0.4849330559372902, "reward_std": 0.16973713226616383, "rewards/accuracy_reward/mean": 0.12276785634458065, "rewards/accuracy_reward/std": 0.31951455771923065, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3621651828289032, "rewards/tag_count_reward/std": 0.12174783833324909, "step": 174 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.75, "completions/mean_length": 995.2500305175781, "completions/mean_terminated_length": 878.4303283691406, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "epoch": 0.052273915316257184, "grad_norm": 0.10317657142877579, "kl": 0.031402587890625, "learning_rate": 1.0875e-05, "loss": 0.0087, "num_tokens": 90938215.0, "reward": 0.4185268133878708, "reward_std": 0.13600957579910755, "rewards/accuracy_reward/mean": 0.07366071408614516, "rewards/accuracy_reward/std": 0.21044625341892242, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3448660746216774, "rewards/tag_count_reward/std": 0.11651718430221081, "step": 175 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7120535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 955.8370819091797, "completions/mean_terminated_length": 777.82470703125, "completions/min_length": 478.5, "completions/min_terminated_length": 478.5, "epoch": 0.052572623403778655, "grad_norm": 0.11877299845218658, "kl": 0.0361328125, "learning_rate": 1.0937500000000002e-05, "loss": 0.0261, "num_tokens": 91436174.0, "reward": 0.5803571715950966, "reward_std": 0.19617865607142448, "rewards/accuracy_reward/mean": 0.19419643096625805, "rewards/accuracy_reward/std": 0.383944995701313, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3861607164144516, "rewards/tag_count_reward/std": 0.12328103370964527, "step": 176 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7299107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 964.6272735595703, "completions/mean_terminated_length": 833.8287963867188, "completions/min_length": 586.0, "completions/min_terminated_length": 586.0, "epoch": 0.052871331491300126, "grad_norm": 0.1147005632519722, "kl": 0.0418701171875, "learning_rate": 1.1000000000000001e-05, "loss": 0.0149, "num_tokens": 91933959.0, "reward": 0.5150669887661934, "reward_std": 0.17673530150204897, "rewards/accuracy_reward/mean": 0.1160714291036129, "rewards/accuracy_reward/std": 0.2645498588681221, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3989955335855484, "rewards/tag_count_reward/std": 0.12070721387863159, "step": 177 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7366071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.5, "completions/mean_length": 956.7210235595703, "completions/mean_terminated_length": 748.1567230224609, "completions/min_length": 495.75, "completions/min_terminated_length": 495.75, "epoch": 0.0531700395788216, "grad_norm": 0.12397121638059616, "kl": 0.0355224609375, "learning_rate": 1.1062500000000001e-05, "loss": 0.0081, "num_tokens": 92427834.0, "reward": 0.5407366305589676, "reward_std": 0.14937912672758102, "rewards/accuracy_reward/mean": 0.16517856647260487, "rewards/accuracy_reward/std": 0.31270249746739864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3755580335855484, "rewards/tag_count_reward/std": 0.11974722519516945, "step": 178 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.75, "completions/mean_length": 944.2567291259766, "completions/mean_terminated_length": 767.5916290283203, "completions/min_length": 546.5, "completions/min_terminated_length": 546.5, "epoch": 0.05346874766634307, "grad_norm": 0.11580542474985123, "kl": 0.037078857421875, "learning_rate": 1.1125000000000001e-05, "loss": 0.0103, "num_tokens": 92916781.0, "reward": 0.5474330708384514, "reward_std": 0.17708738520741463, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.32291877269744873, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3755580335855484, "rewards/tag_count_reward/std": 0.1129635963588953, "step": 179 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 984.8125457763672, "completions/mean_terminated_length": 835.6283874511719, "completions/min_length": 639.5, "completions/min_terminated_length": 639.5, "epoch": 0.05376745575386454, "grad_norm": 0.10279207676649094, "kl": 0.035919189453125, "learning_rate": 1.11875e-05, "loss": 0.0125, "num_tokens": 93433673.0, "reward": 0.419084832072258, "reward_std": 0.11386940348893404, "rewards/accuracy_reward/mean": 0.06026785960420966, "rewards/accuracy_reward/std": 0.19309473782777786, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3588169664144516, "rewards/tag_count_reward/std": 0.12376796081662178, "step": 180 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.75, "completions/mean_length": 939.1004943847656, "completions/mean_terminated_length": 809.7528686523438, "completions/min_length": 542.75, "completions/min_terminated_length": 542.75, "epoch": 0.054066163841386, "grad_norm": 0.1075449138879776, "kl": 0.03912353515625, "learning_rate": 1.125e-05, "loss": 0.019, "num_tokens": 93928246.0, "reward": 0.4352678805589676, "reward_std": 0.14516515098512173, "rewards/accuracy_reward/mean": 0.05580357206054032, "rewards/accuracy_reward/std": 0.21394299902021885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.379464291036129, "rewards/tag_count_reward/std": 0.11787345819175243, "step": 181 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 973.7254943847656, "completions/mean_terminated_length": 795.8388671875, "completions/min_length": 556.5, "completions/min_terminated_length": 556.5, "epoch": 0.054364871928907474, "grad_norm": 0.1190234124660492, "kl": 0.036865234375, "learning_rate": 1.1312500000000002e-05, "loss": 0.0093, "num_tokens": 94436715.0, "reward": 0.4458705484867096, "reward_std": 0.16224403120577335, "rewards/accuracy_reward/mean": 0.08184523973613977, "rewards/accuracy_reward/std": 0.26955367997288704, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3655133917927742, "rewards/tag_count_reward/std": 0.12404039315879345, "step": 182 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7276785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.25, "completions/mean_length": 961.1049499511719, "completions/mean_terminated_length": 807.1834716796875, "completions/min_length": 454.5, "completions/min_terminated_length": 454.5, "epoch": 0.054663580016428945, "grad_norm": 0.10648693889379501, "kl": 0.03985595703125, "learning_rate": 1.1375e-05, "loss": 0.0134, "num_tokens": 94942698.0, "reward": 0.5753348469734192, "reward_std": 0.18207747861742973, "rewards/accuracy_reward/mean": 0.1830357196740806, "rewards/accuracy_reward/std": 0.3584786504507065, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3922991007566452, "rewards/tag_count_reward/std": 0.12425671890377998, "step": 183 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7366071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 971.9486999511719, "completions/mean_terminated_length": 832.8265838623047, "completions/min_length": 466.75, "completions/min_terminated_length": 466.75, "epoch": 0.054962288103950416, "grad_norm": 0.11766397953033447, "kl": 0.0416259765625, "learning_rate": 1.1437500000000002e-05, "loss": 0.0176, "num_tokens": 95453075.0, "reward": 0.5011160895228386, "reward_std": 0.1666440237313509, "rewards/accuracy_reward/mean": 0.1026785708963871, "rewards/accuracy_reward/std": 0.2517468184232712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3984374925494194, "rewards/tag_count_reward/std": 0.1191779337823391, "step": 184 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 969.3638916015625, "completions/mean_terminated_length": 796.6104278564453, "completions/min_length": 487.5, "completions/min_terminated_length": 487.5, "epoch": 0.05526099619147189, "grad_norm": 0.1156662181019783, "kl": 0.04248046875, "learning_rate": 1.15e-05, "loss": 0.0077, "num_tokens": 95960774.0, "reward": 0.537388414144516, "reward_std": 0.18872636556625366, "rewards/accuracy_reward/mean": 0.15624999743886292, "rewards/accuracy_reward/std": 0.3159230016171932, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3811383917927742, "rewards/tag_count_reward/std": 0.12521480582654476, "step": 185 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7142857142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 954.0602874755859, "completions/mean_terminated_length": 791.8924102783203, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "epoch": 0.05555970427899335, "grad_norm": 0.12127648293972015, "kl": 0.044189453125, "learning_rate": 1.1562500000000002e-05, "loss": 0.0143, "num_tokens": 96461729.0, "reward": 0.5725446790456772, "reward_std": 0.2052917256951332, "rewards/accuracy_reward/mean": 0.17410714365541935, "rewards/accuracy_reward/std": 0.3764345496892929, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3984375074505806, "rewards/tag_count_reward/std": 0.12222601287066936, "step": 186 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7924107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 989.6094207763672, "completions/mean_terminated_length": 860.3599548339844, "completions/min_length": 682.0, "completions/min_terminated_length": 682.0, "epoch": 0.05585841236651482, "grad_norm": 0.11813072860240936, "kl": 0.04278564453125, "learning_rate": 1.1625000000000001e-05, "loss": 0.0139, "num_tokens": 96973586.0, "reward": 0.5474330633878708, "reward_std": 0.210675161331892, "rewards/accuracy_reward/mean": 0.1674107164144516, "rewards/accuracy_reward/std": 0.361875269562006, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3800223171710968, "rewards/tag_count_reward/std": 0.12447827868163586, "step": 187 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6919642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 932.4509429931641, "completions/mean_terminated_length": 727.5807037353516, "completions/min_length": 466.25, "completions/min_terminated_length": 466.25, "epoch": 0.05615712045403629, "grad_norm": 0.12972377240657806, "kl": 0.04644775390625, "learning_rate": 1.1687500000000001e-05, "loss": 0.0377, "num_tokens": 97467116.0, "reward": 0.5334821566939354, "reward_std": 0.16876475512981415, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.33460599556565285, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.390625, "rewards/tag_count_reward/std": 0.12373285740613937, "step": 188 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.734375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 964.872802734375, "completions/mean_terminated_length": 800.3487243652344, "completions/min_length": 498.75, "completions/min_terminated_length": 498.75, "epoch": 0.056455828541557763, "grad_norm": 0.12577398121356964, "kl": 0.047119140625, "learning_rate": 1.1750000000000001e-05, "loss": 0.0053, "num_tokens": 97970451.0, "reward": 0.4531250149011612, "reward_std": 0.1801692359149456, "rewards/accuracy_reward/mean": 0.05357142933644354, "rewards/accuracy_reward/std": 0.2096351571381092, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3995535746216774, "rewards/tag_count_reward/std": 0.12029492855072021, "step": 189 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 948.5491485595703, "completions/mean_terminated_length": 837.2315521240234, "completions/min_length": 499.0, "completions/min_terminated_length": 499.0, "epoch": 0.056754536629079234, "grad_norm": 0.11528097093105316, "kl": 0.0472412109375, "learning_rate": 1.18125e-05, "loss": 0.0104, "num_tokens": 98464681.0, "reward": 0.5569196715950966, "reward_std": 0.18393580988049507, "rewards/accuracy_reward/mean": 0.1339285708963871, "rewards/accuracy_reward/std": 0.2856360599398613, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4229910746216774, "rewards/tag_count_reward/std": 0.11224216781556606, "step": 190 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.71875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 966.685302734375, "completions/mean_terminated_length": 827.4260864257812, "completions/min_length": 525.25, "completions/min_terminated_length": 525.25, "epoch": 0.057053244716600705, "grad_norm": 0.14426392316818237, "kl": 0.0494384765625, "learning_rate": 1.1875e-05, "loss": 0.026, "num_tokens": 98965916.0, "reward": 0.5267857387661934, "reward_std": 0.20367446541786194, "rewards/accuracy_reward/mean": 0.12723214668221772, "rewards/accuracy_reward/std": 0.2917295154184103, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3995535671710968, "rewards/tag_count_reward/std": 0.1255202367901802, "step": 191 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6986607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 952.5000305175781, "completions/mean_terminated_length": 794.7068023681641, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.05735195280412217, "grad_norm": 0.12380213290452957, "kl": 0.05401611328125, "learning_rate": 1.1937500000000002e-05, "loss": 0.0248, "num_tokens": 99473276.0, "reward": 0.5161830633878708, "reward_std": 0.20125005766749382, "rewards/accuracy_reward/mean": 0.10714285867288709, "rewards/accuracy_reward/std": 0.2855799086391926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4090401753783226, "rewards/tag_count_reward/std": 0.1179367396980524, "step": 192 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 900.9308471679688, "completions/mean_terminated_length": 756.8345031738281, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "epoch": 0.05765066089164364, "grad_norm": 0.14080430567264557, "kl": 0.06182861328125, "learning_rate": 1.2e-05, "loss": 0.0175, "num_tokens": 99946173.0, "reward": 0.632254496216774, "reward_std": 0.25111738964915276, "rewards/accuracy_reward/mean": 0.19196428265422583, "rewards/accuracy_reward/std": 0.3558385372161865, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4402901828289032, "rewards/tag_count_reward/std": 0.10623250715434551, "step": 193 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6450892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 930.5335235595703, "completions/mean_terminated_length": 756.8126068115234, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "epoch": 0.05794936897916511, "grad_norm": 0.11827249079942703, "kl": 0.0556640625, "learning_rate": 1.2062500000000002e-05, "loss": 0.0096, "num_tokens": 100439020.0, "reward": 0.5385044813156128, "reward_std": 0.2014995440840721, "rewards/accuracy_reward/mean": 0.10714286006987095, "rewards/accuracy_reward/std": 0.30407147109508514, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4313616156578064, "rewards/tag_count_reward/std": 0.11035424657166004, "step": 194 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 929.5357513427734, "completions/mean_terminated_length": 788.4349975585938, "completions/min_length": 357.5, "completions/min_terminated_length": 357.5, "epoch": 0.05824807706668658, "grad_norm": 0.1237206757068634, "kl": 0.054931640625, "learning_rate": 1.2125e-05, "loss": 0.0111, "num_tokens": 100927756.0, "reward": 0.643973246216774, "reward_std": 0.21356228552758694, "rewards/accuracy_reward/mean": 0.2008928582072258, "rewards/accuracy_reward/std": 0.33339405059814453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4430803656578064, "rewards/tag_count_reward/std": 0.10461802966892719, "step": 195 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 924.0982513427734, "completions/mean_terminated_length": 792.7127838134766, "completions/min_length": 416.5, "completions/min_terminated_length": 416.5, "epoch": 0.05854678515420805, "grad_norm": 0.13644754886627197, "kl": 0.06072998046875, "learning_rate": 1.2187500000000001e-05, "loss": 0.0257, "num_tokens": 101414792.0, "reward": 0.5239955633878708, "reward_std": 0.17818961665034294, "rewards/accuracy_reward/mean": 0.08482142817229033, "rewards/accuracy_reward/std": 0.27721893042325974, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4391741082072258, "rewards/tag_count_reward/std": 0.1046811155974865, "step": 196 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 923.1853179931641, "completions/mean_terminated_length": 796.7570495605469, "completions/min_length": 404.5, "completions/min_terminated_length": 404.5, "epoch": 0.05884549324172952, "grad_norm": 0.12470410019159317, "kl": 0.052978515625, "learning_rate": 1.2250000000000001e-05, "loss": 0.0161, "num_tokens": 101908251.0, "reward": 0.5597098469734192, "reward_std": 0.1732801627367735, "rewards/accuracy_reward/mean": 0.1049107126891613, "rewards/accuracy_reward/std": 0.25088198855519295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4547991082072258, "rewards/tag_count_reward/std": 0.09623015485703945, "step": 197 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 950.6406555175781, "completions/mean_terminated_length": 820.5853576660156, "completions/min_length": 506.0, "completions/min_terminated_length": 506.0, "epoch": 0.05914420132925099, "grad_norm": 0.11798709630966187, "kl": 0.04803466796875, "learning_rate": 1.2312500000000001e-05, "loss": -0.0004, "num_tokens": 102404810.0, "reward": 0.5597098469734192, "reward_std": 0.20291677862405777, "rewards/accuracy_reward/mean": 0.11607142817229033, "rewards/accuracy_reward/std": 0.30818191915750504, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4503348171710968, "rewards/tag_count_reward/std": 0.10026746802031994, "step": 198 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 905.2857513427734, "completions/mean_terminated_length": 787.3810577392578, "completions/min_length": 406.5, "completions/min_terminated_length": 406.5, "epoch": 0.05944290941677246, "grad_norm": 0.11614499986171722, "kl": 0.0518798828125, "learning_rate": 1.2375000000000001e-05, "loss": 0.0152, "num_tokens": 102886202.0, "reward": 0.6143973469734192, "reward_std": 0.21117264404892921, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.34857798367738724, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4626116082072258, "rewards/tag_count_reward/std": 0.08917523920536041, "step": 199 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 907.0692291259766, "completions/mean_terminated_length": 758.6032409667969, "completions/min_length": 307.25, "completions/min_terminated_length": 307.25, "epoch": 0.05974161750429393, "grad_norm": 0.11843852698802948, "kl": 0.05108642578125, "learning_rate": 1.24375e-05, "loss": 0.0172, "num_tokens": 103363881.0, "reward": 0.5429687947034836, "reward_std": 0.14658291451632977, "rewards/accuracy_reward/mean": 0.06696428661234677, "rewards/accuracy_reward/std": 0.23423498682677746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4760044664144516, "rewards/tag_count_reward/std": 0.0732798045501113, "step": 200 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 839.6295013427734, "completions/mean_terminated_length": 762.3956146240234, "completions/min_length": 363.75, "completions/min_terminated_length": 363.75, "epoch": 0.0600403255918154, "grad_norm": 0.10632101446390152, "kl": 0.0582275390625, "learning_rate": 1.25e-05, "loss": 0.0037, "num_tokens": 103814627.0, "reward": 0.6316964477300644, "reward_std": 0.17799037136137486, "rewards/accuracy_reward/mean": 0.14322916604578495, "rewards/accuracy_reward/std": 0.3446871489286423, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.038836000952869654, "step": 201 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 869.1295166015625, "completions/mean_terminated_length": 757.5132751464844, "completions/min_length": 296.0, "completions/min_terminated_length": 296.0, "epoch": 0.060339033679336865, "grad_norm": 0.10398299992084503, "kl": 0.0531005859375, "learning_rate": 1.2562500000000002e-05, "loss": -0.008, "num_tokens": 104276781.0, "reward": 0.6283482313156128, "reward_std": 0.12674136273562908, "rewards/accuracy_reward/mean": 0.13392857159487903, "rewards/accuracy_reward/std": 0.2785838972777128, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.034913196228444576, "step": 202 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 886.2299652099609, "completions/mean_terminated_length": 772.4943542480469, "completions/min_length": 403.25, "completions/min_terminated_length": 403.25, "epoch": 0.060637741766858336, "grad_norm": 0.10414111614227295, "kl": 0.0521240234375, "learning_rate": 1.2625e-05, "loss": -0.0174, "num_tokens": 104747188.0, "reward": 0.6902902126312256, "reward_std": 0.15990879386663437, "rewards/accuracy_reward/mean": 0.19866071175783873, "rewards/accuracy_reward/std": 0.30792056769132614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 203 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 858.5089721679688, "completions/mean_terminated_length": 723.4508209228516, "completions/min_length": 326.5, "completions/min_terminated_length": 326.5, "epoch": 0.06093644985437981, "grad_norm": 0.09170660376548767, "kl": 0.0543212890625, "learning_rate": 1.2687500000000002e-05, "loss": -0.0063, "num_tokens": 105206440.0, "reward": 0.6333705633878708, "reward_std": 0.1705980747938156, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.3416338562965393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04000696213915944, "step": 204 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 856.6406707763672, "completions/mean_terminated_length": 769.9617767333984, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.06123515794190128, "grad_norm": 0.0718192607164383, "kl": 0.05242919921875, "learning_rate": 1.275e-05, "loss": 0.003, "num_tokens": 105662935.0, "reward": 0.595982164144516, "reward_std": 0.11877932772040367, "rewards/accuracy_reward/mean": 0.10528273833915591, "rewards/accuracy_reward/std": 0.24044618755578995, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 205 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 874.3393249511719, "completions/mean_terminated_length": 777.3029479980469, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "epoch": 0.06153386602942275, "grad_norm": 0.08607819676399231, "kl": 0.053466796875, "learning_rate": 1.2812500000000001e-05, "loss": 0.0093, "num_tokens": 106138319.0, "reward": 0.5915178954601288, "reward_std": 0.13947529159486294, "rewards/accuracy_reward/mean": 0.09375000023283064, "rewards/accuracy_reward/std": 0.25794622860848904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 206 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 850.419677734375, "completions/mean_terminated_length": 751.2447204589844, "completions/min_length": 249.25, "completions/min_terminated_length": 249.25, "epoch": 0.06183257411694422, "grad_norm": 0.0781857967376709, "kl": 0.0543212890625, "learning_rate": 1.2875000000000001e-05, "loss": 0.0001, "num_tokens": 106593627.0, "reward": 0.6919643133878708, "reward_std": 0.13649022951722145, "rewards/accuracy_reward/mean": 0.1941964328289032, "rewards/accuracy_reward/std": 0.38270673155784607, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 207 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 871.6272888183594, "completions/mean_terminated_length": 760.1822814941406, "completions/min_length": 384.5, "completions/min_terminated_length": 384.5, "epoch": 0.06213128220446568, "grad_norm": 0.08041556179523468, "kl": 0.055419921875, "learning_rate": 1.2937500000000001e-05, "loss": 0.0094, "num_tokens": 107056804.0, "reward": 0.5418527126312256, "reward_std": 0.09945571795105934, "rewards/accuracy_reward/mean": 0.04687500116415322, "rewards/accuracy_reward/std": 0.2009459976106882, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 208 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 879.4531555175781, "completions/mean_terminated_length": 764.2689666748047, "completions/min_length": 352.75, "completions/min_terminated_length": 352.75, "epoch": 0.062429990291987154, "grad_norm": 0.08369731903076172, "kl": 0.05682373046875, "learning_rate": 1.3000000000000001e-05, "loss": 0.0044, "num_tokens": 107521583.0, "reward": 0.5703125298023224, "reward_std": 0.09507914632558823, "rewards/accuracy_reward/mean": 0.0758928582072258, "rewards/accuracy_reward/std": 0.23649326339364052, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 209 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 915.0670166015625, "completions/mean_terminated_length": 810.7487945556641, "completions/min_length": 434.75, "completions/min_terminated_length": 434.75, "epoch": 0.06272869837950862, "grad_norm": 0.10453005135059357, "kl": 0.05194091796875, "learning_rate": 1.30625e-05, "loss": 0.0083, "num_tokens": 108006077.0, "reward": 0.5809151977300644, "reward_std": 0.14871696569025517, "rewards/accuracy_reward/mean": 0.0892857126891613, "rewards/accuracy_reward/std": 0.27560916543006897, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 210 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 914.7321929931641, "completions/mean_terminated_length": 783.2450103759766, "completions/min_length": 424.5, "completions/min_terminated_length": 424.5, "epoch": 0.06302740646703009, "grad_norm": 0.1001533791422844, "kl": 0.05029296875, "learning_rate": 1.3125e-05, "loss": 0.0141, "num_tokens": 108481157.0, "reward": 0.6886160969734192, "reward_std": 0.16502146795392036, "rewards/accuracy_reward/mean": 0.19680059212259948, "rewards/accuracy_reward/std": 0.35534602031111717, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 211 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 927.0536041259766, "completions/mean_terminated_length": 801.9982147216797, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "epoch": 0.06332611455455156, "grad_norm": 0.09537318348884583, "kl": 0.05157470703125, "learning_rate": 1.3187500000000002e-05, "loss": -0.0037, "num_tokens": 108974365.0, "reward": 0.642857164144516, "reward_std": 0.17876439541578293, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.35450392961502075, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.04769135778769851, "step": 212 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 925.1183471679688, "completions/mean_terminated_length": 822.9698333740234, "completions/min_length": 482.75, "completions/min_terminated_length": 482.75, "epoch": 0.06362482264207303, "grad_norm": 0.0943710058927536, "kl": 0.04925537109375, "learning_rate": 1.325e-05, "loss": 0.0084, "num_tokens": 109464610.0, "reward": 0.647879496216774, "reward_std": 0.18584715202450752, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.34374743700027466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.038409143686294556, "step": 213 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 904.3750457763672, "completions/mean_terminated_length": 782.6605377197266, "completions/min_length": 492.5, "completions/min_terminated_length": 492.5, "epoch": 0.0639235307295945, "grad_norm": 0.10323161631822586, "kl": 0.05206298828125, "learning_rate": 1.3312500000000002e-05, "loss": 0.0107, "num_tokens": 109939866.0, "reward": 0.741629496216774, "reward_std": 0.20133361220359802, "rewards/accuracy_reward/mean": 0.2500000037252903, "rewards/accuracy_reward/std": 0.43416785448789597, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04203914059326053, "step": 214 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 927.779052734375, "completions/mean_terminated_length": 814.2761993408203, "completions/min_length": 486.5, "completions/min_terminated_length": 486.5, "epoch": 0.06422223881711597, "grad_norm": 0.08697512000799179, "kl": 0.05316162109375, "learning_rate": 1.3375e-05, "loss": 0.0065, "num_tokens": 110431575.0, "reward": 0.5619419813156128, "reward_std": 0.09863331075757742, "rewards/accuracy_reward/mean": 0.0691964270081371, "rewards/accuracy_reward/std": 0.2193977702409029, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04015073133632541, "step": 215 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 889.9754791259766, "completions/mean_terminated_length": 794.1879730224609, "completions/min_length": 408.75, "completions/min_terminated_length": 408.75, "epoch": 0.06452094690463744, "grad_norm": 0.10038100928068161, "kl": 0.05352783203125, "learning_rate": 1.3437500000000001e-05, "loss": 0.0114, "num_tokens": 110900732.0, "reward": 0.6534598618745804, "reward_std": 0.12789439130574465, "rewards/accuracy_reward/mean": 0.15848214086145163, "rewards/accuracy_reward/std": 0.349880114197731, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.03309101238846779, "step": 216 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 884.4754791259766, "completions/mean_terminated_length": 788.0633392333984, "completions/min_length": 347.75, "completions/min_terminated_length": 347.75, "epoch": 0.06481965499215891, "grad_norm": 0.10807402431964874, "kl": 0.052490234375, "learning_rate": 1.3500000000000001e-05, "loss": 0.0163, "num_tokens": 111364353.0, "reward": 0.6462053805589676, "reward_std": 0.19765276834368706, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.33256392925977707, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.030178462620824575, "step": 217 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 864.9375610351562, "completions/mean_terminated_length": 775.9094390869141, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "epoch": 0.06511836307968039, "grad_norm": 0.10291379690170288, "kl": 0.05731201171875, "learning_rate": 1.3562500000000001e-05, "loss": 0.0101, "num_tokens": 111824789.0, "reward": 0.6428571790456772, "reward_std": 0.21636931598186493, "rewards/accuracy_reward/mean": 0.14508928917348385, "rewards/accuracy_reward/std": 0.3352561742067337, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 218 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 903.1518096923828, "completions/mean_terminated_length": 803.3918914794922, "completions/min_length": 396.75, "completions/min_terminated_length": 396.75, "epoch": 0.06541707116720186, "grad_norm": 0.09277545660734177, "kl": 0.05279541015625, "learning_rate": 1.3625e-05, "loss": 0.0047, "num_tokens": 112303897.0, "reward": 0.6344866305589676, "reward_std": 0.15667996928095818, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3450803607702255, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 219 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 897.8594207763672, "completions/mean_terminated_length": 806.2245178222656, "completions/min_length": 359.25, "completions/min_terminated_length": 359.25, "epoch": 0.06571577925472333, "grad_norm": 0.09425082802772522, "kl": 0.05706787109375, "learning_rate": 1.36875e-05, "loss": 0.0063, "num_tokens": 112785114.0, "reward": 0.561941996216774, "reward_std": 0.09499471448361874, "rewards/accuracy_reward/mean": 0.06696428498253226, "rewards/accuracy_reward/std": 0.19947116822004318, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.030101283453404903, "step": 220 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 856.7098693847656, "completions/mean_terminated_length": 769.6739807128906, "completions/min_length": 316.5, "completions/min_terminated_length": 316.5, "epoch": 0.06601448734224478, "grad_norm": 0.06764274835586548, "kl": 0.05816650390625, "learning_rate": 1.375e-05, "loss": -0.0067, "num_tokens": 113245496.0, "reward": 0.6383928805589676, "reward_std": 0.10782013949938118, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.3398372530937195, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 221 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 828.0826263427734, "completions/mean_terminated_length": 748.3119812011719, "completions/min_length": 303.25, "completions/min_terminated_length": 303.25, "epoch": 0.06631319542976626, "grad_norm": 0.08981853723526001, "kl": 0.057373046875, "learning_rate": 1.3812500000000002e-05, "loss": 0.0057, "num_tokens": 113687245.0, "reward": 0.6523437798023224, "reward_std": 0.12151985615491867, "rewards/accuracy_reward/mean": 0.1610863134264946, "rewards/accuracy_reward/std": 0.3620873764157295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 222 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 879.6741485595703, "completions/mean_terminated_length": 774.5095672607422, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "epoch": 0.06661190351728773, "grad_norm": 0.1140490472316742, "kl": 0.0570068359375, "learning_rate": 1.3875e-05, "loss": 0.0131, "num_tokens": 114148587.0, "reward": 0.690848246216774, "reward_std": 0.1370067559182644, "rewards/accuracy_reward/mean": 0.1964285671710968, "rewards/accuracy_reward/std": 0.38928838074207306, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 223 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 904.279052734375, "completions/mean_terminated_length": 811.366943359375, "completions/min_length": 382.75, "completions/min_terminated_length": 382.75, "epoch": 0.0669106116048092, "grad_norm": 0.08499237149953842, "kl": 0.0537109375, "learning_rate": 1.3937500000000002e-05, "loss": 0.0093, "num_tokens": 114625368.0, "reward": 0.5809152126312256, "reward_std": 0.11837916448712349, "rewards/accuracy_reward/mean": 0.08482143096625805, "rewards/accuracy_reward/std": 0.2609933912754059, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937574505806, "rewards/tag_count_reward/std": 0.024776804260909557, "step": 224 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 909.7545013427734, "completions/mean_terminated_length": 819.1996765136719, "completions/min_length": 417.75, "completions/min_terminated_length": 417.75, "epoch": 0.06720931969233067, "grad_norm": 0.08959444612264633, "kl": 0.053466796875, "learning_rate": 1.4e-05, "loss": 0.0029, "num_tokens": 115102714.0, "reward": 0.6361607387661934, "reward_std": 0.15712925232946873, "rewards/accuracy_reward/mean": 0.14285713993012905, "rewards/accuracy_reward/std": 0.2889712527394295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.037598448805511, "step": 225 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 925.4397735595703, "completions/mean_terminated_length": 823.9197235107422, "completions/min_length": 461.75, "completions/min_terminated_length": 461.75, "epoch": 0.06750802777985214, "grad_norm": 0.06625261157751083, "kl": 0.04925537109375, "learning_rate": 1.4062500000000001e-05, "loss": 0.003, "num_tokens": 115588143.0, "reward": 0.6194196790456772, "reward_std": 0.12634137459099293, "rewards/accuracy_reward/mean": 0.12053571222350001, "rewards/accuracy_reward/std": 0.2980985939502716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839253783226, "rewards/tag_count_reward/std": 0.008314208127558231, "step": 226 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 892.4286193847656, "completions/mean_terminated_length": 779.8479309082031, "completions/min_length": 330.75, "completions/min_terminated_length": 330.75, "epoch": 0.06780673586737361, "grad_norm": 0.07066900283098221, "kl": 0.05426025390625, "learning_rate": 1.4125000000000003e-05, "loss": 0.01, "num_tokens": 116054415.0, "reward": 0.6378348395228386, "reward_std": 0.08546552900224924, "rewards/accuracy_reward/mean": 0.1428571492433548, "rewards/accuracy_reward/std": 0.2268921583890915, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.029007501434534788, "step": 227 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 924.6339721679688, "completions/mean_terminated_length": 809.6992950439453, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "epoch": 0.06810544395489508, "grad_norm": 0.08517643809318542, "kl": 0.051025390625, "learning_rate": 1.4187500000000001e-05, "loss": 0.0067, "num_tokens": 116537035.0, "reward": 0.6534598469734192, "reward_std": 0.14243419282138348, "rewards/accuracy_reward/mean": 0.16071428917348385, "rewards/accuracy_reward/std": 0.3501931056380272, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.037192290648818016, "step": 228 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 934.9598693847656, "completions/mean_terminated_length": 825.6371307373047, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "epoch": 0.06840415204241655, "grad_norm": 0.07103745639324188, "kl": 0.051025390625, "learning_rate": 1.425e-05, "loss": 0.0077, "num_tokens": 117024633.0, "reward": 0.5546875298023224, "reward_std": 0.09272357309237123, "rewards/accuracy_reward/mean": 0.058035715483129025, "rewards/accuracy_reward/std": 0.19889093935489655, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 229 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 915.5803985595703, "completions/mean_terminated_length": 810.5484161376953, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.06870286012993802, "grad_norm": 0.09589124470949173, "kl": 0.05169677734375, "learning_rate": 1.43125e-05, "loss": 0.0022, "num_tokens": 117512525.0, "reward": 0.6417410969734192, "reward_std": 0.13865421526134014, "rewards/accuracy_reward/mean": 0.14955357322469354, "rewards/accuracy_reward/std": 0.31072045490145683, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.042415475472807884, "step": 230 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 899.419677734375, "completions/mean_terminated_length": 781.7910461425781, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "epoch": 0.0690015682174595, "grad_norm": 0.0981728732585907, "kl": 0.0543212890625, "learning_rate": 1.4375e-05, "loss": 0.0122, "num_tokens": 117991769.0, "reward": 0.7488839626312256, "reward_std": 0.1731477491557598, "rewards/accuracy_reward/mean": 0.2522321417927742, "rewards/accuracy_reward/std": 0.431743323802948, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517761349678, "rewards/tag_count_reward/std": 0.024942624382674694, "step": 231 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 936.8861999511719, "completions/mean_terminated_length": 829.9402160644531, "completions/min_length": 450.25, "completions/min_terminated_length": 450.25, "epoch": 0.06930027630498095, "grad_norm": 0.08879068493843079, "kl": 0.05181884765625, "learning_rate": 1.4437500000000002e-05, "loss": 0.005, "num_tokens": 118484246.0, "reward": 0.5507812649011612, "reward_std": 0.13477075845003128, "rewards/accuracy_reward/mean": 0.055803571827709675, "rewards/accuracy_reward/std": 0.22998185455799103, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 232 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 898.8705902099609, "completions/mean_terminated_length": 794.1792297363281, "completions/min_length": 465.5, "completions/min_terminated_length": 465.5, "epoch": 0.06959898439250242, "grad_norm": 0.16003923118114471, "kl": 0.06280517578125, "learning_rate": 1.45e-05, "loss": 0.0126, "num_tokens": 118954636.0, "reward": 0.7248884290456772, "reward_std": 0.17078049667179585, "rewards/accuracy_reward/mean": 0.22991071012802422, "rewards/accuracy_reward/std": 0.35800000838935375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 233 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 910.9486999511719, "completions/mean_terminated_length": 803.4420013427734, "completions/min_length": 455.25, "completions/min_terminated_length": 455.25, "epoch": 0.06989769248002389, "grad_norm": 0.12566198408603668, "kl": 0.05340576171875, "learning_rate": 1.4562500000000002e-05, "loss": 0.009, "num_tokens": 119432005.0, "reward": 0.628348246216774, "reward_std": 0.16469855047762394, "rewards/accuracy_reward/mean": 0.14285714086145163, "rewards/accuracy_reward/std": 0.3176770359277725, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.058124168775975704, "step": 234 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6495535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 955.0290679931641, "completions/mean_terminated_length": 829.78955078125, "completions/min_length": 535.5, "completions/min_terminated_length": 535.5, "epoch": 0.07019640056754536, "grad_norm": 0.13929277658462524, "kl": 0.059814453125, "learning_rate": 1.4625e-05, "loss": 0.0316, "num_tokens": 119935682.0, "reward": 0.560825914144516, "reward_std": 0.2124109137803316, "rewards/accuracy_reward/mean": 0.12276785634458065, "rewards/accuracy_reward/std": 0.2708515077829361, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4380580335855484, "rewards/tag_count_reward/std": 0.108165068551898, "step": 235 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6540178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 945.5268249511719, "completions/mean_terminated_length": 804.2130126953125, "completions/min_length": 483.25, "completions/min_terminated_length": 483.25, "epoch": 0.07049510865506683, "grad_norm": 0.13199833035469055, "kl": 0.06048583984375, "learning_rate": 1.4687500000000001e-05, "loss": 0.0268, "num_tokens": 120437150.0, "reward": 0.558035746216774, "reward_std": 0.1805508229881525, "rewards/accuracy_reward/mean": 0.12276785937137902, "rewards/accuracy_reward/std": 0.22700184397399426, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4352678507566452, "rewards/tag_count_reward/std": 0.10956954769790173, "step": 236 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 943.9777221679688, "completions/mean_terminated_length": 807.4625244140625, "completions/min_length": 537.25, "completions/min_terminated_length": 537.25, "epoch": 0.0707938167425883, "grad_norm": 0.11556994915008545, "kl": 0.05322265625, "learning_rate": 1.4750000000000003e-05, "loss": 0.0097, "num_tokens": 120926084.0, "reward": 0.6629464477300644, "reward_std": 0.22639114409685135, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3718726485967636, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4732142835855484, "rewards/tag_count_reward/std": 0.07696101628243923, "step": 237 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 887.685302734375, "completions/mean_terminated_length": 778.0865783691406, "completions/min_length": 410.25, "completions/min_terminated_length": 410.25, "epoch": 0.07109252483010978, "grad_norm": 0.09073647111654282, "kl": 0.05419921875, "learning_rate": 1.4812500000000001e-05, "loss": 0.0115, "num_tokens": 121395031.0, "reward": 0.630580373108387, "reward_std": 0.10296996869146824, "rewards/accuracy_reward/mean": 0.1383928544819355, "rewards/accuracy_reward/std": 0.2711306996643543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.042415475472807884, "step": 238 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 901.6920013427734, "completions/mean_terminated_length": 811.7801361083984, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "epoch": 0.07139123291763125, "grad_norm": 0.08629193902015686, "kl": 0.05316162109375, "learning_rate": 1.4875000000000002e-05, "loss": -0.0015, "num_tokens": 121867661.0, "reward": 0.681919664144516, "reward_std": 0.11120004020631313, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.37236588448286057, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952479079366, "step": 239 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 883.9174499511719, "completions/mean_terminated_length": 786.3617095947266, "completions/min_length": 377.25, "completions/min_terminated_length": 377.25, "epoch": 0.07168994100515272, "grad_norm": 0.09052885323762894, "kl": 0.0557861328125, "learning_rate": 1.49375e-05, "loss": 0.0007, "num_tokens": 122329448.0, "reward": 0.6199777126312256, "reward_std": 0.10467994632199407, "rewards/accuracy_reward/mean": 0.12537202495150268, "rewards/accuracy_reward/std": 0.301959540694952, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.494977667927742, "rewards/tag_count_reward/std": 0.03507901635020971, "step": 240 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 921.7790679931641, "completions/mean_terminated_length": 823.7772979736328, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "epoch": 0.07198864909267419, "grad_norm": 0.0869554728269577, "kl": 0.051513671875, "learning_rate": 1.5000000000000002e-05, "loss": 0.006, "num_tokens": 122814821.0, "reward": 0.7187500447034836, "reward_std": 0.1970645673573017, "rewards/accuracy_reward/mean": 0.22098214365541935, "rewards/accuracy_reward/std": 0.39921512454748154, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 241 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 880.8995971679688, "completions/mean_terminated_length": 784.9844970703125, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "epoch": 0.07228735718019566, "grad_norm": 0.08246652036905289, "kl": 0.05419921875, "learning_rate": 1.5062500000000002e-05, "loss": 0.0039, "num_tokens": 123285736.0, "reward": 0.7087053954601288, "reward_std": 0.14340642467141151, "rewards/accuracy_reward/mean": 0.20982142630964518, "rewards/accuracy_reward/std": 0.37813447415828705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 242 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 943.3415679931641, "completions/mean_terminated_length": 824.1140289306641, "completions/min_length": 291.0, "completions/min_terminated_length": 291.0, "epoch": 0.07258606526771712, "grad_norm": 0.06726032495498657, "kl": 0.05010986328125, "learning_rate": 1.5125e-05, "loss": 0.0011, "num_tokens": 123793105.0, "reward": 0.5708705633878708, "reward_std": 0.07668199902400374, "rewards/accuracy_reward/mean": 0.0736607164144516, "rewards/accuracy_reward/std": 0.23934857919812202, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02147206850349903, "step": 243 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 905.3236999511719, "completions/mean_terminated_length": 799.1026763916016, "completions/min_length": 419.5, "completions/min_terminated_length": 419.5, "epoch": 0.07288477335523859, "grad_norm": 0.09691514819860458, "kl": 0.05413818359375, "learning_rate": 1.5187500000000002e-05, "loss": 0.013, "num_tokens": 124270402.0, "reward": 0.6344866454601288, "reward_std": 0.1861027292907238, "rewards/accuracy_reward/mean": 0.1480654734186828, "rewards/accuracy_reward/std": 0.3294735848903656, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 244 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 913.7768402099609, "completions/mean_terminated_length": 799.4425354003906, "completions/min_length": 475.25, "completions/min_terminated_length": 475.25, "epoch": 0.07318348144276006, "grad_norm": 0.07472251355648041, "kl": 0.05352783203125, "learning_rate": 1.525e-05, "loss": 0.0037, "num_tokens": 124746382.0, "reward": 0.5039062723517418, "reward_std": 0.06013355916365981, "rewards/accuracy_reward/mean": 0.01116071455180645, "rewards/accuracy_reward/std": 0.08779112622141838, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 245 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 903.6228179931641, "completions/mean_terminated_length": 802.154296875, "completions/min_length": 421.75, "completions/min_terminated_length": 421.75, "epoch": 0.07348218953028153, "grad_norm": 0.1004365012049675, "kl": 0.05389404296875, "learning_rate": 1.5312500000000003e-05, "loss": 0.0164, "num_tokens": 125223013.0, "reward": 0.6445312798023224, "reward_std": 0.16196623258292675, "rewards/accuracy_reward/mean": 0.15178571455180645, "rewards/accuracy_reward/std": 0.35716280341148376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03955313144251704, "step": 246 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 905.9353179931641, "completions/mean_terminated_length": 807.1304779052734, "completions/min_length": 350.5, "completions/min_terminated_length": 350.5, "epoch": 0.073780897617803, "grad_norm": 0.08986783772706985, "kl": 0.05322265625, "learning_rate": 1.5375e-05, "loss": 0.0105, "num_tokens": 125702136.0, "reward": 0.6774553805589676, "reward_std": 0.1382384318858385, "rewards/accuracy_reward/mean": 0.18303571362048388, "rewards/accuracy_reward/std": 0.35567473620176315, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 247 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 902.2232513427734, "completions/mean_terminated_length": 815.1166839599609, "completions/min_length": 379.5, "completions/min_terminated_length": 379.5, "epoch": 0.07407960570532447, "grad_norm": 0.09666698426008224, "kl": 0.06011962890625, "learning_rate": 1.54375e-05, "loss": -0.0018, "num_tokens": 126177932.0, "reward": 0.6177455633878708, "reward_std": 0.1848999299108982, "rewards/accuracy_reward/mean": 0.1205357164144516, "rewards/accuracy_reward/std": 0.3158179074525833, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.01845060009509325, "step": 248 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 904.8236999511719, "completions/mean_terminated_length": 800.7349090576172, "completions/min_length": 385.75, "completions/min_terminated_length": 385.75, "epoch": 0.07437831379284594, "grad_norm": 0.07742788642644882, "kl": 0.055908203125, "learning_rate": 1.55e-05, "loss": 0.0123, "num_tokens": 126651069.0, "reward": 0.6406250298023224, "reward_std": 0.13374298438429832, "rewards/accuracy_reward/mean": 0.14285714412108064, "rewards/accuracy_reward/std": 0.32470739260315895, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 249 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 914.0000457763672, "completions/mean_terminated_length": 817.6403045654297, "completions/min_length": 470.75, "completions/min_terminated_length": 470.75, "epoch": 0.07467702188036741, "grad_norm": 0.09111052751541138, "kl": 0.055908203125, "learning_rate": 1.55625e-05, "loss": 0.0058, "num_tokens": 127136685.0, "reward": 0.5697544813156128, "reward_std": 0.11167778261005878, "rewards/accuracy_reward/mean": 0.07589285634458065, "rewards/accuracy_reward/std": 0.2271420881152153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 250 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 907.0647735595703, "completions/mean_terminated_length": 812.5003509521484, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.07497572996788888, "grad_norm": 0.08941596746444702, "kl": 0.0537109375, "learning_rate": 1.5625e-05, "loss": 0.0017, "num_tokens": 127629226.0, "reward": 0.660714328289032, "reward_std": 0.17306693084537983, "rewards/accuracy_reward/mean": 0.16517857182770967, "rewards/accuracy_reward/std": 0.34406960755586624, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.025947765447199345, "step": 251 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 953.9062957763672, "completions/mean_terminated_length": 852.9417419433594, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "epoch": 0.07527443805541036, "grad_norm": 0.0992312952876091, "kl": 0.05657958984375, "learning_rate": 1.5687500000000002e-05, "loss": 0.0051, "num_tokens": 128129200.0, "reward": 0.553571455180645, "reward_std": 0.10240319184958935, "rewards/accuracy_reward/mean": 0.06473214272409678, "rewards/accuracy_reward/std": 0.20786359906196594, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 252 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 901.3995971679688, "completions/mean_terminated_length": 813.471435546875, "completions/min_length": 443.25, "completions/min_terminated_length": 443.25, "epoch": 0.07557314614293181, "grad_norm": 0.09366612136363983, "kl": 0.05609130859375, "learning_rate": 1.575e-05, "loss": 0.0061, "num_tokens": 128600739.0, "reward": 0.7109375298023224, "reward_std": 0.1614409014582634, "rewards/accuracy_reward/mean": 0.21651785261929035, "rewards/accuracy_reward/std": 0.3860658332705498, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03192346729338169, "step": 253 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 904.8661041259766, "completions/mean_terminated_length": 802.2284698486328, "completions/min_length": 352.25, "completions/min_terminated_length": 352.25, "epoch": 0.07587185423045328, "grad_norm": 0.1054823100566864, "kl": 0.05657958984375, "learning_rate": 1.58125e-05, "loss": 0.0185, "num_tokens": 129081079.0, "reward": 0.6964285969734192, "reward_std": 0.17722555808722973, "rewards/accuracy_reward/mean": 0.20535714365541935, "rewards/accuracy_reward/std": 0.38829725980758667, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.046403173357248306, "step": 254 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 938.6763916015625, "completions/mean_terminated_length": 818.1036834716797, "completions/min_length": 543.25, "completions/min_terminated_length": 543.25, "epoch": 0.07617056231797475, "grad_norm": 0.0840764045715332, "kl": 0.0543212890625, "learning_rate": 1.5875e-05, "loss": 0.0099, "num_tokens": 129574166.0, "reward": 0.6590402126312256, "reward_std": 0.12307834811508656, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3560606688261032, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 255 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 921.8504943847656, "completions/mean_terminated_length": 813.6371612548828, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "epoch": 0.07646927040549623, "grad_norm": 0.09032971411943436, "kl": 0.0574951171875, "learning_rate": 1.59375e-05, "loss": 0.0002, "num_tokens": 130061683.0, "reward": 0.5948660969734192, "reward_std": 0.11755084432661533, "rewards/accuracy_reward/mean": 0.1071428582072258, "rewards/accuracy_reward/std": 0.3016791120171547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.055453699082136154, "step": 256 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 915.0022735595703, "completions/mean_terminated_length": 804.4660339355469, "completions/min_length": 360.25, "completions/min_terminated_length": 360.25, "epoch": 0.0767679784930177, "grad_norm": 0.13214394450187683, "kl": 0.06414794921875, "learning_rate": 1.6000000000000003e-05, "loss": 0.002, "num_tokens": 130548612.0, "reward": 0.6785714477300644, "reward_std": 0.20000974833965302, "rewards/accuracy_reward/mean": 0.1852678544819355, "rewards/accuracy_reward/std": 0.38577643781900406, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 257 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 907.1920013427734, "completions/mean_terminated_length": 796.9862518310547, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "epoch": 0.07706668658053917, "grad_norm": 0.10886357724666595, "kl": 0.05902099609375, "learning_rate": 1.60625e-05, "loss": 0.0078, "num_tokens": 131030570.0, "reward": 0.6478794813156128, "reward_std": 0.1528464201837778, "rewards/accuracy_reward/mean": 0.16555059514939785, "rewards/accuracy_reward/std": 0.3415764719247818, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05902230739593506, "step": 258 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 910.2098693847656, "completions/mean_terminated_length": 808.1572570800781, "completions/min_length": 375.25, "completions/min_terminated_length": 375.25, "epoch": 0.07736539466806064, "grad_norm": 0.1252042055130005, "kl": 0.06890869140625, "learning_rate": 1.6125000000000002e-05, "loss": 0.0138, "num_tokens": 131516248.0, "reward": 0.6104910969734192, "reward_std": 0.19118185713887215, "rewards/accuracy_reward/mean": 0.1205357126891613, "rewards/accuracy_reward/std": 0.31887752935290337, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046347017865628004, "step": 259 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 966.3527221679688, "completions/mean_terminated_length": 864.7072296142578, "completions/min_length": 552.0, "completions/min_terminated_length": 552.0, "epoch": 0.07766410275558211, "grad_norm": 0.11505576968193054, "kl": 0.0618896484375, "learning_rate": 1.61875e-05, "loss": 0.0066, "num_tokens": 132016022.0, "reward": 0.6685268133878708, "reward_std": 0.14341634511947632, "rewards/accuracy_reward/mean": 0.18303571036085486, "rewards/accuracy_reward/std": 0.31481390073895454, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05717269331216812, "step": 260 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 947.8973846435547, "completions/mean_terminated_length": 836.2854919433594, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.07796281084310358, "grad_norm": 0.10806356370449066, "kl": 0.0640869140625, "learning_rate": 1.6250000000000002e-05, "loss": 0.0008, "num_tokens": 132516888.0, "reward": 0.5987723469734192, "reward_std": 0.19617370888590813, "rewards/accuracy_reward/mean": 0.1116071455180645, "rewards/accuracy_reward/std": 0.30400026589632034, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 261 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 939.4732666015625, "completions/mean_terminated_length": 824.9604949951172, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.07826151893062505, "grad_norm": 0.13670742511749268, "kl": 0.06414794921875, "learning_rate": 1.6312500000000003e-05, "loss": 0.0259, "num_tokens": 133014300.0, "reward": 0.690848246216774, "reward_std": 0.21214341185986996, "rewards/accuracy_reward/mean": 0.221726197283715, "rewards/accuracy_reward/std": 0.37483031302690506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06633441615849733, "step": 262 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 940.2634429931641, "completions/mean_terminated_length": 834.7153778076172, "completions/min_length": 350.75, "completions/min_terminated_length": 350.75, "epoch": 0.07856022701814652, "grad_norm": 0.1087612584233284, "kl": 0.06634521484375, "learning_rate": 1.6375e-05, "loss": 0.0025, "num_tokens": 133511826.0, "reward": 0.5156250074505806, "reward_std": 0.10283753019757569, "rewards/accuracy_reward/mean": 0.03571428684517741, "rewards/accuracy_reward/std": 0.1514688991010189, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4799107164144516, "rewards/tag_count_reward/std": 0.061602677684277296, "step": 263 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 914.9085235595703, "completions/mean_terminated_length": 811.6699066162109, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "epoch": 0.07885893510566798, "grad_norm": 0.09926111996173859, "kl": 0.06207275390625, "learning_rate": 1.6437500000000003e-05, "loss": 0.0136, "num_tokens": 133994457.0, "reward": 0.6255580633878708, "reward_std": 0.15352597832679749, "rewards/accuracy_reward/mean": 0.13169643096625805, "rewards/accuracy_reward/std": 0.3220762200653553, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 264 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44642857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 900.8616485595703, "completions/mean_terminated_length": 801.6281127929688, "completions/min_length": 387.0, "completions/min_terminated_length": 387.0, "epoch": 0.07915764319318945, "grad_norm": 0.09469010680913925, "kl": 0.0640869140625, "learning_rate": 1.65e-05, "loss": 0.007, "num_tokens": 134471931.0, "reward": 0.5418527126312256, "reward_std": 0.056244557024911046, "rewards/accuracy_reward/mean": 0.04687500186264515, "rewards/accuracy_reward/std": 0.1756660807877779, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 265 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 925.654052734375, "completions/mean_terminated_length": 818.9288177490234, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "epoch": 0.07945635128071092, "grad_norm": 0.10603159666061401, "kl": 0.0634765625, "learning_rate": 1.6562500000000003e-05, "loss": 0.0104, "num_tokens": 134957936.0, "reward": 0.6914062798023224, "reward_std": 0.19356437027454376, "rewards/accuracy_reward/mean": 0.19866071734577417, "rewards/accuracy_reward/std": 0.36867717653512955, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03606722131371498, "step": 266 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 906.5134429931641, "completions/mean_terminated_length": 802.8831787109375, "completions/min_length": 366.25, "completions/min_terminated_length": 366.25, "epoch": 0.07975505936823239, "grad_norm": 0.08600489050149918, "kl": 0.0611572265625, "learning_rate": 1.6625e-05, "loss": 0.0129, "num_tokens": 135435414.0, "reward": 0.5591518133878708, "reward_std": 0.09767957031726837, "rewards/accuracy_reward/mean": 0.06473214435391128, "rewards/accuracy_reward/std": 0.21220859326422215, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.03659330680966377, "step": 267 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 894.8058319091797, "completions/mean_terminated_length": 778.6928863525391, "completions/min_length": 401.25, "completions/min_terminated_length": 401.25, "epoch": 0.08005376745575386, "grad_norm": 0.08700212836265564, "kl": 0.06182861328125, "learning_rate": 1.6687500000000002e-05, "loss": 0.012, "num_tokens": 135912559.0, "reward": 0.5797991305589676, "reward_std": 0.09367741364985704, "rewards/accuracy_reward/mean": 0.08705357322469354, "rewards/accuracy_reward/std": 0.23174136504530907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03541599866002798, "step": 268 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 889.6786041259766, "completions/mean_terminated_length": 798.8929595947266, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.08035247554327533, "grad_norm": 0.12109698355197906, "kl": 0.07318115234375, "learning_rate": 1.675e-05, "loss": 0.0111, "num_tokens": 136378431.0, "reward": 0.5714285969734192, "reward_std": 0.10300229117274284, "rewards/accuracy_reward/mean": 0.07589285727590322, "rewards/accuracy_reward/std": 0.2575908489525318, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 269 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 938.0089721679688, "completions/mean_terminated_length": 828.6136016845703, "completions/min_length": 356.25, "completions/min_terminated_length": 356.25, "epoch": 0.0806511836307968, "grad_norm": 0.10147041827440262, "kl": 0.05975341796875, "learning_rate": 1.6812500000000002e-05, "loss": 0.0087, "num_tokens": 136874947.0, "reward": 0.7003348618745804, "reward_std": 0.16873571649193764, "rewards/accuracy_reward/mean": 0.2075892831198871, "rewards/accuracy_reward/std": 0.3554828092455864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 270 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 919.1339721679688, "completions/mean_terminated_length": 795.4097290039062, "completions/min_length": 426.5, "completions/min_terminated_length": 426.5, "epoch": 0.08094989171831828, "grad_norm": 0.0931464284658432, "kl": 0.05926513671875, "learning_rate": 1.6875e-05, "loss": 0.0164, "num_tokens": 137361519.0, "reward": 0.6132812798023224, "reward_std": 0.16770772077143192, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3270059674978256, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.049205650109797716, "step": 271 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 942.4286041259766, "completions/mean_terminated_length": 822.2698516845703, "completions/min_length": 486.25, "completions/min_terminated_length": 486.25, "epoch": 0.08124859980583975, "grad_norm": 0.09309928119182587, "kl": 0.05810546875, "learning_rate": 1.6937500000000002e-05, "loss": 0.0092, "num_tokens": 137852671.0, "reward": 0.575334832072258, "reward_std": 0.09124994510784745, "rewards/accuracy_reward/mean": 0.08705357275903225, "rewards/accuracy_reward/std": 0.25390681996941566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05116795934736729, "step": 272 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 953.7611999511719, "completions/mean_terminated_length": 851.4303436279297, "completions/min_length": 570.25, "completions/min_terminated_length": 570.25, "epoch": 0.08154730789336122, "grad_norm": 0.11543793976306915, "kl": 0.0604248046875, "learning_rate": 1.7e-05, "loss": 0.0155, "num_tokens": 138354660.0, "reward": 0.6065848618745804, "reward_std": 0.2124401405453682, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.325543824583292, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848171710968, "rewards/tag_count_reward/std": 0.06475724838674068, "step": 273 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 963.3705749511719, "completions/mean_terminated_length": 849.5315856933594, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "epoch": 0.08184601598088269, "grad_norm": 0.10985057055950165, "kl": 0.0599365234375, "learning_rate": 1.70625e-05, "loss": 0.0086, "num_tokens": 138856202.0, "reward": 0.6601562798023224, "reward_std": 0.19932325184345245, "rewards/accuracy_reward/mean": 0.17336309235543013, "rewards/accuracy_reward/std": 0.33982883766293526, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.044885930605232716, "step": 274 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6473214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 950.5156555175781, "completions/mean_terminated_length": 834.770751953125, "completions/min_length": 560.75, "completions/min_terminated_length": 560.75, "epoch": 0.08214472406840415, "grad_norm": 0.11103551089763641, "kl": 0.061767578125, "learning_rate": 1.7125e-05, "loss": 0.0134, "num_tokens": 139349345.0, "reward": 0.6607143208384514, "reward_std": 0.14450253546237946, "rewards/accuracy_reward/mean": 0.18080356949940324, "rewards/accuracy_reward/std": 0.3520756922662258, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4799107164144516, "rewards/tag_count_reward/std": 0.06654847972095013, "step": 275 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 960.779052734375, "completions/mean_terminated_length": 857.9615936279297, "completions/min_length": 501.75, "completions/min_terminated_length": 501.75, "epoch": 0.08244343215592562, "grad_norm": 0.11823670566082001, "kl": 0.061279296875, "learning_rate": 1.71875e-05, "loss": 0.0096, "num_tokens": 139853662.0, "reward": 0.636160746216774, "reward_std": 0.1881253980100155, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.3460090383887291, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06356978882104158, "step": 276 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 962.6629943847656, "completions/mean_terminated_length": 865.3460998535156, "completions/min_length": 599.25, "completions/min_terminated_length": 599.25, "epoch": 0.08274214024344709, "grad_norm": 0.09902060031890869, "kl": 0.056884765625, "learning_rate": 1.7250000000000003e-05, "loss": 0.0038, "num_tokens": 140355879.0, "reward": 0.5837053656578064, "reward_std": 0.13292028196156025, "rewards/accuracy_reward/mean": 0.09374999906867743, "rewards/accuracy_reward/std": 0.2646343410015106, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 277 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 946.5067291259766, "completions/mean_terminated_length": 820.5579986572266, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.08304084833096856, "grad_norm": 0.1033862978219986, "kl": 0.05938720703125, "learning_rate": 1.73125e-05, "loss": 0.016, "num_tokens": 140857274.0, "reward": 0.6283482313156128, "reward_std": 0.1308161709457636, "rewards/accuracy_reward/mean": 0.1339285741560161, "rewards/accuracy_reward/std": 0.3046744279563427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.034913196228444576, "step": 278 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 946.7768249511719, "completions/mean_terminated_length": 847.103759765625, "completions/min_length": 493.25, "completions/min_terminated_length": 493.25, "epoch": 0.08333955641849003, "grad_norm": 0.09969841688871384, "kl": 0.05987548828125, "learning_rate": 1.7375000000000002e-05, "loss": 0.0058, "num_tokens": 141354838.0, "reward": 0.6534598469734192, "reward_std": 0.1468743085861206, "rewards/accuracy_reward/mean": 0.16071428265422583, "rewards/accuracy_reward/std": 0.3510269597172737, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 279 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 930.8214721679688, "completions/mean_terminated_length": 831.1888427734375, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "epoch": 0.0836382645060115, "grad_norm": 0.07617359608411789, "kl": 0.05975341796875, "learning_rate": 1.74375e-05, "loss": 0.0126, "num_tokens": 141842070.0, "reward": 0.5781250298023224, "reward_std": 0.10892337234690785, "rewards/accuracy_reward/mean": 0.08035714481957257, "rewards/accuracy_reward/std": 0.20273848250508308, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 280 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 938.8281707763672, "completions/mean_terminated_length": 835.6844177246094, "completions/min_length": 532.75, "completions/min_terminated_length": 532.75, "epoch": 0.08393697259353297, "grad_norm": 0.07539110630750656, "kl": 0.06085205078125, "learning_rate": 1.7500000000000002e-05, "loss": 0.0116, "num_tokens": 142337209.0, "reward": 0.5747768133878708, "reward_std": 0.108728788793087, "rewards/accuracy_reward/mean": 0.08035714365541935, "rewards/accuracy_reward/std": 0.26894252747297287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.029416739474982023, "step": 281 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 951.4643249511719, "completions/mean_terminated_length": 856.7067108154297, "completions/min_length": 528.75, "completions/min_terminated_length": 528.75, "epoch": 0.08423568068105444, "grad_norm": 0.09524964541196823, "kl": 0.05743408203125, "learning_rate": 1.7562500000000003e-05, "loss": 0.0111, "num_tokens": 142840217.0, "reward": 0.6065848469734192, "reward_std": 0.11361717246472836, "rewards/accuracy_reward/mean": 0.11383928474970162, "rewards/accuracy_reward/std": 0.288604324683547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04000696213915944, "step": 282 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 921.3236846923828, "completions/mean_terminated_length": 801.0401458740234, "completions/min_length": 374.5, "completions/min_terminated_length": 374.5, "epoch": 0.08453438876857591, "grad_norm": 0.07509851455688477, "kl": 0.06024169921875, "learning_rate": 1.7625e-05, "loss": 0.0117, "num_tokens": 143328634.0, "reward": 0.6294643133878708, "reward_std": 0.12366172298789024, "rewards/accuracy_reward/mean": 0.13169642724096775, "rewards/accuracy_reward/std": 0.33847856521606445, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 283 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 892.2120971679688, "completions/mean_terminated_length": 807.6772613525391, "completions/min_length": 371.25, "completions/min_terminated_length": 371.25, "epoch": 0.08483309685609738, "grad_norm": 0.09250865131616592, "kl": 0.0604248046875, "learning_rate": 1.7687500000000003e-05, "loss": 0.0086, "num_tokens": 143803593.0, "reward": 0.6741071790456772, "reward_std": 0.21371568366885185, "rewards/accuracy_reward/mean": 0.18526786006987095, "rewards/accuracy_reward/std": 0.38178159296512604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 284 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 914.2388763427734, "completions/mean_terminated_length": 804.9389038085938, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "epoch": 0.08513180494361886, "grad_norm": 0.06485940515995026, "kl": 0.05743408203125, "learning_rate": 1.775e-05, "loss": 0.0098, "num_tokens": 144287236.0, "reward": 0.604910746216774, "reward_std": 0.08833541441708803, "rewards/accuracy_reward/mean": 0.10491071501746774, "rewards/accuracy_reward/std": 0.28657131269574165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.5, "rewards/tag_count_reward/std": 0.0, "step": 285 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 877.4196624755859, "completions/mean_terminated_length": 789.4548950195312, "completions/min_length": 432.5, "completions/min_terminated_length": 432.5, "epoch": 0.08543051303114031, "grad_norm": 0.06452818214893341, "kl": 0.0599365234375, "learning_rate": 1.7812500000000003e-05, "loss": 0.0083, "num_tokens": 144747904.0, "reward": 0.6512276977300644, "reward_std": 0.12799482606351376, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.36049800366163254, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 286 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 950.0692596435547, "completions/mean_terminated_length": 842.8635406494141, "completions/min_length": 470.5, "completions/min_terminated_length": 470.5, "epoch": 0.08572922111866178, "grad_norm": 0.06801079958677292, "kl": 0.05645751953125, "learning_rate": 1.7875e-05, "loss": 0.0088, "num_tokens": 145245439.0, "reward": 0.6551339477300644, "reward_std": 0.10819721897132695, "rewards/accuracy_reward/mean": 0.15848214365541935, "rewards/accuracy_reward/std": 0.35860126465559006, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.023462072014808655, "step": 287 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 890.8817291259766, "completions/mean_terminated_length": 794.5123901367188, "completions/min_length": 359.5, "completions/min_terminated_length": 359.5, "epoch": 0.08602792920618325, "grad_norm": 0.058809004724025726, "kl": 0.06085205078125, "learning_rate": 1.7937500000000002e-05, "loss": 0.0074, "num_tokens": 145725722.0, "reward": 0.5496652126312256, "reward_std": 0.08842142671346664, "rewards/accuracy_reward/mean": 0.051339286379516125, "rewards/accuracy_reward/std": 0.20638976246118546, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.010136391967535019, "step": 288 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 937.8080749511719, "completions/mean_terminated_length": 821.3047027587891, "completions/min_length": 460.0, "completions/min_terminated_length": 460.0, "epoch": 0.08632663729370472, "grad_norm": 0.07721570134162903, "kl": 0.0579833984375, "learning_rate": 1.8e-05, "loss": 0.0071, "num_tokens": 146228244.0, "reward": 0.5736607313156128, "reward_std": 0.13443170674145222, "rewards/accuracy_reward/mean": 0.07589285960420966, "rewards/accuracy_reward/std": 0.2331656962633133, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 289 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 911.8795166015625, "completions/mean_terminated_length": 784.8662414550781, "completions/min_length": 450.75, "completions/min_terminated_length": 450.75, "epoch": 0.0866253453812262, "grad_norm": 0.08900684118270874, "kl": 0.05767822265625, "learning_rate": 1.8062500000000002e-05, "loss": 0.0185, "num_tokens": 146705454.0, "reward": 0.7276785969734192, "reward_std": 0.1985054761171341, "rewards/accuracy_reward/mean": 0.23214285634458065, "rewards/accuracy_reward/std": 0.3874766603112221, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.031776280142366886, "step": 290 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 891.7299499511719, "completions/mean_terminated_length": 777.9908905029297, "completions/min_length": 415.5, "completions/min_terminated_length": 415.5, "epoch": 0.08692405346874767, "grad_norm": 0.08251228928565979, "kl": 0.0626220703125, "learning_rate": 1.8125e-05, "loss": 0.0179, "num_tokens": 147180229.0, "reward": 0.6328125298023224, "reward_std": 0.13331842608749866, "rewards/accuracy_reward/mean": 0.1480654808692634, "rewards/accuracy_reward/std": 0.3252801150083542, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.025787058286368847, "step": 291 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 911.8370971679688, "completions/mean_terminated_length": 807.0271148681641, "completions/min_length": 451.75, "completions/min_terminated_length": 451.75, "epoch": 0.08722276155626914, "grad_norm": 0.07312064617872238, "kl": 0.0594482421875, "learning_rate": 1.81875e-05, "loss": 0.0079, "num_tokens": 147663372.0, "reward": 0.6573660969734192, "reward_std": 0.13649551384150982, "rewards/accuracy_reward/mean": 0.1584821450524032, "rewards/accuracy_reward/std": 0.33524206280708313, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839253783226, "rewards/tag_count_reward/std": 0.008314208127558231, "step": 292 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 943.2924346923828, "completions/mean_terminated_length": 806.8170776367188, "completions/min_length": 407.5, "completions/min_terminated_length": 407.5, "epoch": 0.08752146964379061, "grad_norm": 0.08745778352022171, "kl": 0.0601806640625, "learning_rate": 1.825e-05, "loss": 0.0084, "num_tokens": 148160607.0, "reward": 0.6149553805589676, "reward_std": 0.13713014964014292, "rewards/accuracy_reward/mean": 0.12053571711294353, "rewards/accuracy_reward/std": 0.26875751093029976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.029416739474982023, "step": 293 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6919642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 975.1027221679688, "completions/mean_terminated_length": 865.6416931152344, "completions/min_length": 515.5, "completions/min_terminated_length": 515.5, "epoch": 0.08782017773131208, "grad_norm": 0.08687349408864975, "kl": 0.05731201171875, "learning_rate": 1.83125e-05, "loss": 0.0077, "num_tokens": 148667053.0, "reward": 0.6450893133878708, "reward_std": 0.17810197547078133, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.355087973177433, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.028279099613428116, "step": 294 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7142857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 968.9821929931641, "completions/mean_terminated_length": 830.6391906738281, "completions/min_length": 480.0, "completions/min_terminated_length": 480.0, "epoch": 0.08811888581883355, "grad_norm": 0.08421216160058975, "kl": 0.05853271484375, "learning_rate": 1.8375e-05, "loss": 0.0042, "num_tokens": 149165573.0, "reward": 0.5719866305589676, "reward_std": 0.15223068930208683, "rewards/accuracy_reward/mean": 0.08035714388825, "rewards/accuracy_reward/std": 0.24848077818751335, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.042492654640227556, "step": 295 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 929.8839721679688, "completions/mean_terminated_length": 817.8202667236328, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "epoch": 0.08841759390635502, "grad_norm": 0.0786745622754097, "kl": 0.0594482421875, "learning_rate": 1.84375e-05, "loss": 0.0084, "num_tokens": 149651617.0, "reward": 0.6841518133878708, "reward_std": 0.11600574292242527, "rewards/accuracy_reward/mean": 0.1897321487776935, "rewards/accuracy_reward/std": 0.3578050211071968, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 296 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5803571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 943.6495971679688, "completions/mean_terminated_length": 834.6385955810547, "completions/min_length": 506.5, "completions/min_terminated_length": 506.5, "epoch": 0.08871630199387648, "grad_norm": 0.08140948414802551, "kl": 0.0609130859375, "learning_rate": 1.8500000000000002e-05, "loss": 0.0103, "num_tokens": 150150756.0, "reward": 0.5340402126312256, "reward_std": 0.10367047786712646, "rewards/accuracy_reward/mean": 0.04017857229337096, "rewards/accuracy_reward/std": 0.17540039494633675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 297 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 938.9397735595703, "completions/mean_terminated_length": 815.213623046875, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "epoch": 0.08901501008139795, "grad_norm": 0.08461079001426697, "kl": 0.06103515625, "learning_rate": 1.85625e-05, "loss": 0.0076, "num_tokens": 150639641.0, "reward": 0.6004464477300644, "reward_std": 0.1307860140223056, "rewards/accuracy_reward/mean": 0.10491071688011289, "rewards/accuracy_reward/std": 0.2780657708644867, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 298 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 921.4085388183594, "completions/mean_terminated_length": 820.5858612060547, "completions/min_length": 506.75, "completions/min_terminated_length": 506.75, "epoch": 0.08931371816891942, "grad_norm": 0.08868726342916489, "kl": 0.060791015625, "learning_rate": 1.8625000000000002e-05, "loss": 0.0101, "num_tokens": 151118480.0, "reward": 0.6227678954601288, "reward_std": 0.18383627757430077, "rewards/accuracy_reward/mean": 0.1272321455180645, "rewards/accuracy_reward/std": 0.3213687464594841, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 299 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4821428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 905.8884429931641, "completions/mean_terminated_length": 798.5582580566406, "completions/min_length": 433.75, "completions/min_terminated_length": 433.75, "epoch": 0.08961242625644089, "grad_norm": 0.09216473996639252, "kl": 0.06036376953125, "learning_rate": 1.86875e-05, "loss": 0.0109, "num_tokens": 151595614.0, "reward": 0.7695312798023224, "reward_std": 0.1680205799639225, "rewards/accuracy_reward/mean": 0.27455356903374195, "rewards/accuracy_reward/std": 0.4248223081231117, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.029593830928206444, "step": 300 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6361607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 963.9620971679688, "completions/mean_terminated_length": 878.1924438476562, "completions/min_length": 564.25, "completions/min_terminated_length": 564.25, "epoch": 0.08991113434396236, "grad_norm": 0.07369610667228699, "kl": 0.0579833984375, "learning_rate": 1.8750000000000002e-05, "loss": 0.0049, "num_tokens": 152106973.0, "reward": 0.5770089626312256, "reward_std": 0.110868064686656, "rewards/accuracy_reward/mean": 0.08035714272409678, "rewards/accuracy_reward/std": 0.2672761604189873, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 301 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 901.1652221679688, "completions/mean_terminated_length": 805.3997039794922, "completions/min_length": 448.5, "completions/min_terminated_length": 448.5, "epoch": 0.09020984243148383, "grad_norm": 0.09371527284383774, "kl": 0.06024169921875, "learning_rate": 1.8812500000000003e-05, "loss": 0.0031, "num_tokens": 152583911.0, "reward": 0.7371652126312256, "reward_std": 0.14649919606745243, "rewards/accuracy_reward/mean": 0.2410714291036129, "rewards/accuracy_reward/std": 0.4136992320418358, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 302 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 925.1161193847656, "completions/mean_terminated_length": 822.8049926757812, "completions/min_length": 340.25, "completions/min_terminated_length": 340.25, "epoch": 0.0905085505190053, "grad_norm": 0.0815277174115181, "kl": 0.06121826171875, "learning_rate": 1.8875e-05, "loss": 0.0006, "num_tokens": 153075227.0, "reward": 0.616629496216774, "reward_std": 0.12093103490769863, "rewards/accuracy_reward/mean": 0.12053571688011289, "rewards/accuracy_reward/std": 0.30301380529999733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.02676480822265148, "step": 303 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 940.1964721679688, "completions/mean_terminated_length": 851.4378814697266, "completions/min_length": 507.0, "completions/min_terminated_length": 507.0, "epoch": 0.09080725860652678, "grad_norm": 0.08774513751268387, "kl": 0.05865478515625, "learning_rate": 1.8937500000000003e-05, "loss": 0.0074, "num_tokens": 153567091.0, "reward": 0.6339285969734192, "reward_std": 0.16792599484324455, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.25504179671406746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 304 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 903.7031555175781, "completions/mean_terminated_length": 818.8864135742188, "completions/min_length": 482.75, "completions/min_terminated_length": 482.75, "epoch": 0.09110596669404825, "grad_norm": 0.07815393060445786, "kl": 0.06036376953125, "learning_rate": 1.9e-05, "loss": 0.0069, "num_tokens": 154043054.0, "reward": 0.662388414144516, "reward_std": 0.11685299873352051, "rewards/accuracy_reward/mean": 0.16517856949940324, "rewards/accuracy_reward/std": 0.33625612780451775, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 305 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 939.7678985595703, "completions/mean_terminated_length": 830.8941650390625, "completions/min_length": 475.5, "completions/min_terminated_length": 475.5, "epoch": 0.09140467478156972, "grad_norm": 0.0782933235168457, "kl": 0.05950927734375, "learning_rate": 1.9062500000000003e-05, "loss": 0.0073, "num_tokens": 154539718.0, "reward": 0.7578125447034836, "reward_std": 0.1636082895565778, "rewards/accuracy_reward/mean": 0.2611607201397419, "rewards/accuracy_reward/std": 0.3231764957308769, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 306 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 882.0312805175781, "completions/mean_terminated_length": 803.4490356445312, "completions/min_length": 462.5, "completions/min_terminated_length": 462.5, "epoch": 0.09170338286909117, "grad_norm": 0.11045334488153458, "kl": 0.0673828125, "learning_rate": 1.9125000000000004e-05, "loss": 0.0059, "num_tokens": 155002964.0, "reward": 0.6356026977300644, "reward_std": 0.18724675849080086, "rewards/accuracy_reward/mean": 0.1406249962747097, "rewards/accuracy_reward/std": 0.3434927389025688, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 307 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 938.8772888183594, "completions/mean_terminated_length": 834.7214202880859, "completions/min_length": 462.5, "completions/min_terminated_length": 462.5, "epoch": 0.09200209095661264, "grad_norm": 0.09008724987506866, "kl": 0.0599365234375, "learning_rate": 1.9187500000000002e-05, "loss": 0.0151, "num_tokens": 155492093.0, "reward": 0.746651828289032, "reward_std": 0.20644745789468288, "rewards/accuracy_reward/mean": 0.25446428917348385, "rewards/accuracy_reward/std": 0.40087562799453735, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04111117962747812, "step": 308 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49107142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 906.4062805175781, "completions/mean_terminated_length": 808.14111328125, "completions/min_length": 387.75, "completions/min_terminated_length": 387.75, "epoch": 0.09230079904413412, "grad_norm": 0.066852867603302, "kl": 0.06201171875, "learning_rate": 1.925e-05, "loss": 0.0037, "num_tokens": 155962643.0, "reward": 0.6579241305589676, "reward_std": 0.08443371881730855, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.27477775514125824, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 309 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 923.8660888671875, "completions/mean_terminated_length": 799.4669647216797, "completions/min_length": 435.75, "completions/min_terminated_length": 435.75, "epoch": 0.09259950713165559, "grad_norm": 0.09097030758857727, "kl": 0.06085205078125, "learning_rate": 1.9312500000000002e-05, "loss": 0.0121, "num_tokens": 156447383.0, "reward": 0.5742187798023224, "reward_std": 0.13849111832678318, "rewards/accuracy_reward/mean": 0.0803571417927742, "rewards/accuracy_reward/std": 0.22378426790237427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 310 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5870535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 949.7745971679688, "completions/mean_terminated_length": 847.3614807128906, "completions/min_length": 553.75, "completions/min_terminated_length": 553.75, "epoch": 0.09289821521917706, "grad_norm": 0.09180508553981781, "kl": 0.06182861328125, "learning_rate": 1.9375e-05, "loss": 0.0122, "num_tokens": 156945666.0, "reward": 0.671316996216774, "reward_std": 0.16695085540413857, "rewards/accuracy_reward/mean": 0.17633928637951612, "rewards/accuracy_reward/std": 0.3527998849749565, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.034184794407337904, "step": 311 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 935.3437805175781, "completions/mean_terminated_length": 812.91650390625, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "epoch": 0.09319692330669853, "grad_norm": 0.09913420677185059, "kl": 0.067138671875, "learning_rate": 1.94375e-05, "loss": 0.0044, "num_tokens": 157437788.0, "reward": 0.7159598469734192, "reward_std": 0.1951974555850029, "rewards/accuracy_reward/mean": 0.2232142835855484, "rewards/accuracy_reward/std": 0.40389879792928696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 312 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6674107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 948.3192443847656, "completions/mean_terminated_length": 795.6558380126953, "completions/min_length": 468.0, "completions/min_terminated_length": 468.0, "epoch": 0.09349563139422, "grad_norm": 0.09660059958696365, "kl": 0.06243896484375, "learning_rate": 1.95e-05, "loss": 0.0081, "num_tokens": 157931483.0, "reward": 0.5446428805589676, "reward_std": 0.09187128581106663, "rewards/accuracy_reward/mean": 0.0565476194024086, "rewards/accuracy_reward/std": 0.2160363681614399, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 313 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.71875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 968.4152221679688, "completions/mean_terminated_length": 827.2008209228516, "completions/min_length": 485.25, "completions/min_terminated_length": 485.25, "epoch": 0.09379433948174147, "grad_norm": 0.0808100625872612, "kl": 0.0595703125, "learning_rate": 1.95625e-05, "loss": 0.0003, "num_tokens": 158442213.0, "reward": 0.6060268133878708, "reward_std": 0.12097598239779472, "rewards/accuracy_reward/mean": 0.11569940205663443, "rewards/accuracy_reward/std": 0.3050566613674164, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 314 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 948.6451263427734, "completions/mean_terminated_length": 818.3987274169922, "completions/min_length": 461.75, "completions/min_terminated_length": 461.75, "epoch": 0.09409304756926294, "grad_norm": 0.08957840502262115, "kl": 0.05804443359375, "learning_rate": 1.9625e-05, "loss": 0.0086, "num_tokens": 158936230.0, "reward": 0.6222098469734192, "reward_std": 0.11788473464548588, "rewards/accuracy_reward/mean": 0.12946428824216127, "rewards/accuracy_reward/std": 0.32288575172424316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03865890856832266, "step": 315 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 959.3527221679688, "completions/mean_terminated_length": 844.9192810058594, "completions/min_length": 545.0, "completions/min_terminated_length": 545.0, "epoch": 0.09439175565678441, "grad_norm": 0.11620970815420151, "kl": 0.06903076171875, "learning_rate": 1.96875e-05, "loss": 0.0073, "num_tokens": 159435796.0, "reward": 0.6941964626312256, "reward_std": 0.22171101719141006, "rewards/accuracy_reward/mean": 0.2087053619325161, "rewards/accuracy_reward/std": 0.38886934518814087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 316 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 956.2745971679688, "completions/mean_terminated_length": 852.1145935058594, "completions/min_length": 413.25, "completions/min_terminated_length": 413.25, "epoch": 0.09469046374430588, "grad_norm": 0.09306168556213379, "kl": 0.0626220703125, "learning_rate": 1.9750000000000002e-05, "loss": 0.0069, "num_tokens": 159932607.0, "reward": 0.6143973469734192, "reward_std": 0.08439734857529402, "rewards/accuracy_reward/mean": 0.12053571082651615, "rewards/accuracy_reward/std": 0.2719883993268013, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 317 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7566964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 978.9978179931641, "completions/mean_terminated_length": 837.3599395751953, "completions/min_length": 480.25, "completions/min_terminated_length": 480.25, "epoch": 0.09498917183182734, "grad_norm": 0.09222240000963211, "kl": 0.064208984375, "learning_rate": 1.98125e-05, "loss": 0.0043, "num_tokens": 160445502.0, "reward": 0.5825892984867096, "reward_std": 0.13703910075128078, "rewards/accuracy_reward/mean": 0.0892857126891613, "rewards/accuracy_reward/std": 0.28422709181904793, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.040545567870140076, "step": 318 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 949.3482666015625, "completions/mean_terminated_length": 849.5796508789062, "completions/min_length": 602.25, "completions/min_terminated_length": 602.25, "epoch": 0.09528787991934881, "grad_norm": 0.10850472003221512, "kl": 0.068603515625, "learning_rate": 1.9875000000000002e-05, "loss": 0.0003, "num_tokens": 160952234.0, "reward": 0.6679687798023224, "reward_std": 0.20146328955888748, "rewards/accuracy_reward/mean": 0.1741071417927742, "rewards/accuracy_reward/std": 0.3701821640133858, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.036084157414734364, "step": 319 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 981.7812957763672, "completions/mean_terminated_length": 881.4560089111328, "completions/min_length": 560.0, "completions/min_terminated_length": 560.0, "epoch": 0.09558658800687028, "grad_norm": 0.10152841359376907, "kl": 0.0614013671875, "learning_rate": 1.99375e-05, "loss": 0.0047, "num_tokens": 161455896.0, "reward": 0.601004496216774, "reward_std": 0.1606297381222248, "rewards/accuracy_reward/mean": 0.10714285727590322, "rewards/accuracy_reward/std": 0.3046824410557747, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 320 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 945.4040679931641, "completions/mean_terminated_length": 840.1191253662109, "completions/min_length": 488.25, "completions/min_terminated_length": 488.25, "epoch": 0.09588529609439175, "grad_norm": 0.08903592824935913, "kl": 0.06622314453125, "learning_rate": 2e-05, "loss": 0.011, "num_tokens": 161949405.0, "reward": 0.636160746216774, "reward_std": 0.17573999613523483, "rewards/accuracy_reward/mean": 0.14062500465661287, "rewards/accuracy_reward/std": 0.3247722238302231, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02827909868210554, "step": 321 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 952.0112152099609, "completions/mean_terminated_length": 831.7237091064453, "completions/min_length": 465.25, "completions/min_terminated_length": 465.25, "epoch": 0.09618400418191322, "grad_norm": 0.08101941645145416, "kl": 0.0648193359375, "learning_rate": 1.999999405044161e-05, "loss": 0.0065, "num_tokens": 162441794.0, "reward": 0.6339285969734192, "reward_std": 0.11097302893176675, "rewards/accuracy_reward/mean": 0.1383928544819355, "rewards/accuracy_reward/std": 0.2836579233407974, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 322 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7366071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 985.529052734375, "completions/mean_terminated_length": 878.0447387695312, "completions/min_length": 645.25, "completions/min_terminated_length": 645.25, "epoch": 0.0964827122694347, "grad_norm": 0.0682254284620285, "kl": 0.06640625, "learning_rate": 1.999997620177352e-05, "loss": 0.0064, "num_tokens": 162962943.0, "reward": 0.5518973469734192, "reward_std": 0.04785410175099969, "rewards/accuracy_reward/mean": 0.058035716880112886, "rewards/accuracy_reward/std": 0.1858060024678707, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03323819860816002, "step": 323 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 945.0245971679688, "completions/mean_terminated_length": 849.5995941162109, "completions/min_length": 550.0, "completions/min_terminated_length": 550.0, "epoch": 0.09678142035695617, "grad_norm": 0.10407717525959015, "kl": 0.0648193359375, "learning_rate": 1.999994645401697e-05, "loss": 0.0128, "num_tokens": 163459082.0, "reward": 0.6529018133878708, "reward_std": 0.1817287839949131, "rewards/accuracy_reward/mean": 0.1629464253783226, "rewards/accuracy_reward/std": 0.3544710502028465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 324 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 924.6674652099609, "completions/mean_terminated_length": 811.7135162353516, "completions/min_length": 513.0, "completions/min_terminated_length": 513.0, "epoch": 0.09708012844447764, "grad_norm": 0.0926470011472702, "kl": 0.068603515625, "learning_rate": 1.9999904807207348e-05, "loss": 0.0128, "num_tokens": 163940133.0, "reward": 0.6395089626312256, "reward_std": 0.1440278198570013, "rewards/accuracy_reward/mean": 0.14508928917348385, "rewards/accuracy_reward/std": 0.31794293969869614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.034913196228444576, "step": 325 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 941.2053985595703, "completions/mean_terminated_length": 853.2623443603516, "completions/min_length": 572.5, "completions/min_terminated_length": 572.5, "epoch": 0.09737883653199911, "grad_norm": 0.12360385805368423, "kl": 0.07464599609375, "learning_rate": 1.999985126139422e-05, "loss": 0.0001, "num_tokens": 164439825.0, "reward": 0.6428571492433548, "reward_std": 0.1396855041384697, "rewards/accuracy_reward/mean": 0.1495535746216774, "rewards/accuracy_reward/std": 0.35192733258008957, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 326 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 942.5736846923828, "completions/mean_terminated_length": 814.751708984375, "completions/min_length": 506.75, "completions/min_terminated_length": 506.75, "epoch": 0.09767754461952058, "grad_norm": 0.10764935612678528, "kl": 0.0706787109375, "learning_rate": 1.9999785816641293e-05, "loss": 0.0098, "num_tokens": 164939794.0, "reward": 0.5937500298023224, "reward_std": 0.1313736028969288, "rewards/accuracy_reward/mean": 0.10044642793945968, "rewards/accuracy_reward/std": 0.233310978859663, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.039929782040417194, "step": 327 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 932.9196929931641, "completions/mean_terminated_length": 838.1570281982422, "completions/min_length": 488.5, "completions/min_terminated_length": 488.5, "epoch": 0.09797625270704205, "grad_norm": 0.10082797706127167, "kl": 0.0703125, "learning_rate": 1.999970847302645e-05, "loss": 0.0101, "num_tokens": 165437998.0, "reward": 0.5831473618745804, "reward_std": 0.15423789992928505, "rewards/accuracy_reward/mean": 0.08928571455180645, "rewards/accuracy_reward/std": 0.282597441226244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 328 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 936.1719207763672, "completions/mean_terminated_length": 822.8879699707031, "completions/min_length": 489.25, "completions/min_terminated_length": 489.25, "epoch": 0.0982749607945635, "grad_norm": 0.09247886389493942, "kl": 0.0709228515625, "learning_rate": 1.9999619230641714e-05, "loss": 0.0164, "num_tokens": 165935259.0, "reward": 0.624441996216774, "reward_std": 0.14323920384049416, "rewards/accuracy_reward/mean": 0.1294642831198871, "rewards/accuracy_reward/std": 0.3191547691822052, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 329 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 947.8393249511719, "completions/mean_terminated_length": 845.9975433349609, "completions/min_length": 432.5, "completions/min_terminated_length": 432.5, "epoch": 0.09857366888208498, "grad_norm": 0.07166247814893723, "kl": 0.0689697265625, "learning_rate": 1.999951808959328e-05, "loss": -0.0035, "num_tokens": 166433011.0, "reward": 0.607700914144516, "reward_std": 0.1394304633140564, "rewards/accuracy_reward/mean": 0.11160714598372579, "rewards/accuracy_reward/std": 0.2762480229139328, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 330 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 924.2478179931641, "completions/mean_terminated_length": 830.5872497558594, "completions/min_length": 558.25, "completions/min_terminated_length": 558.25, "epoch": 0.09887237696960645, "grad_norm": 0.07257506996393204, "kl": 0.0718994140625, "learning_rate": 1.99994050500015e-05, "loss": 0.0058, "num_tokens": 166914882.0, "reward": 0.6222098469734192, "reward_std": 0.10742677003145218, "rewards/accuracy_reward/mean": 0.12276785401627421, "rewards/accuracy_reward/std": 0.3125529810786247, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 331 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 954.0201263427734, "completions/mean_terminated_length": 862.1217651367188, "completions/min_length": 502.75, "completions/min_terminated_length": 502.75, "epoch": 0.09917108505712792, "grad_norm": 0.08005926012992859, "kl": 0.06768798828125, "learning_rate": 1.9999280112000875e-05, "loss": 0.0027, "num_tokens": 167414507.0, "reward": 0.5468750298023224, "reward_std": 0.08703942131251097, "rewards/accuracy_reward/mean": 0.05133928661234677, "rewards/accuracy_reward/std": 0.17290333472192287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027692769188433886, "step": 332 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 941.3214721679688, "completions/mean_terminated_length": 817.7191925048828, "completions/min_length": 499.0, "completions/min_terminated_length": 499.0, "epoch": 0.09946979314464939, "grad_norm": 0.08289790898561478, "kl": 0.070068359375, "learning_rate": 1.999914327574007e-05, "loss": 0.0008, "num_tokens": 167910107.0, "reward": 0.580357164144516, "reward_std": 0.10844961926341057, "rewards/accuracy_reward/mean": 0.08482142770662904, "rewards/accuracy_reward/std": 0.2620198391377926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027692769188433886, "step": 333 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 954.1674499511719, "completions/mean_terminated_length": 848.1080627441406, "completions/min_length": 512.25, "completions/min_terminated_length": 512.25, "epoch": 0.09976850123217086, "grad_norm": 0.10093905031681061, "kl": 0.06793212890625, "learning_rate": 1.9998994541381914e-05, "loss": 0.0029, "num_tokens": 168407190.0, "reward": 0.6992187649011612, "reward_std": 0.2615770921111107, "rewards/accuracy_reward/mean": 0.2075892835855484, "rewards/accuracy_reward/std": 0.3787209466099739, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 334 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 958.6496124267578, "completions/mean_terminated_length": 855.5418548583984, "completions/min_length": 625.25, "completions/min_terminated_length": 625.25, "epoch": 0.10006720931969233, "grad_norm": 0.1072990745306015, "kl": 0.073486328125, "learning_rate": 1.9998833909103385e-05, "loss": 0.0104, "num_tokens": 168907769.0, "reward": 0.5848214477300644, "reward_std": 0.13093553972430527, "rewards/accuracy_reward/mean": 0.09598214458674192, "rewards/accuracy_reward/std": 0.2868843451142311, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04981599189341068, "step": 335 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6428571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 962.9018249511719, "completions/mean_terminated_length": 863.6905212402344, "completions/min_length": 593.5, "completions/min_terminated_length": 593.5, "epoch": 0.1003659174072138, "grad_norm": 0.11019807308912277, "kl": 0.0728759765625, "learning_rate": 1.9998661379095622e-05, "loss": 0.0133, "num_tokens": 169414029.0, "reward": 0.6194196715950966, "reward_std": 0.14380819629877806, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.2727745473384857, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.05895264819264412, "step": 336 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7299107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 977.4643402099609, "completions/mean_terminated_length": 859.3486328125, "completions/min_length": 551.75, "completions/min_terminated_length": 551.75, "epoch": 0.10066462549473527, "grad_norm": 0.09765532612800598, "kl": 0.06884765625, "learning_rate": 1.9998476951563914e-05, "loss": 0.0064, "num_tokens": 169914877.0, "reward": 0.5446428954601288, "reward_std": 0.1017148895189166, "rewards/accuracy_reward/mean": 0.0558035708963871, "rewards/accuracy_reward/std": 0.18867479264736176, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982259005308, "step": 337 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 930.3036041259766, "completions/mean_terminated_length": 831.982666015625, "completions/min_length": 592.0, "completions/min_terminated_length": 592.0, "epoch": 0.10096333358225675, "grad_norm": 0.080986388027668, "kl": 0.0660400390625, "learning_rate": 1.999828062672772e-05, "loss": 0.0062, "num_tokens": 170404885.0, "reward": 0.688058078289032, "reward_std": 0.15313223865814507, "rewards/accuracy_reward/mean": 0.19196429289877415, "rewards/accuracy_reward/std": 0.2938542738556862, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 338 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 965.6585388183594, "completions/mean_terminated_length": 867.3169708251953, "completions/min_length": 524.25, "completions/min_terminated_length": 524.25, "epoch": 0.10126204166977822, "grad_norm": 0.09012776613235474, "kl": 0.0716552734375, "learning_rate": 1.9998072404820648e-05, "loss": -0.0035, "num_tokens": 170912044.0, "reward": 0.6132812798023224, "reward_std": 0.13325428403913975, "rewards/accuracy_reward/mean": 0.12053571455180645, "rewards/accuracy_reward/std": 0.3130129389464855, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.038913180120289326, "step": 339 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 958.7433319091797, "completions/mean_terminated_length": 852.7803955078125, "completions/min_length": 529.25, "completions/min_terminated_length": 529.25, "epoch": 0.10156074975729967, "grad_norm": 0.09352005273103714, "kl": 0.0672607421875, "learning_rate": 1.9997852286090466e-05, "loss": 0.0065, "num_tokens": 171410777.0, "reward": 0.6389509290456772, "reward_std": 0.1549353487789631, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.34508902207016945, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 340 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6361607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 957.8683471679688, "completions/mean_terminated_length": 842.5918884277344, "completions/min_length": 504.5, "completions/min_terminated_length": 504.5, "epoch": 0.10185945784482114, "grad_norm": 0.08329658955335617, "kl": 0.064697265625, "learning_rate": 1.999762027079909e-05, "loss": 0.0049, "num_tokens": 171912046.0, "reward": 0.7081473469734192, "reward_std": 0.16957601089961827, "rewards/accuracy_reward/mean": 0.2120535746216774, "rewards/accuracy_reward/std": 0.40155111253261566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.026178478728979826, "step": 341 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 936.0960388183594, "completions/mean_terminated_length": 838.7666015625, "completions/min_length": 472.25, "completions/min_terminated_length": 472.25, "epoch": 0.10215816593234262, "grad_norm": 0.08077766746282578, "kl": 0.06646728515625, "learning_rate": 1.9997376359222604e-05, "loss": 0.013, "num_tokens": 172404729.0, "reward": 0.6612723618745804, "reward_std": 0.12077282927930355, "rewards/accuracy_reward/mean": 0.1651785692665726, "rewards/accuracy_reward/std": 0.2974387314170599, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.02676480822265148, "step": 342 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 942.6473541259766, "completions/mean_terminated_length": 837.3531799316406, "completions/min_length": 513.75, "completions/min_terminated_length": 513.75, "epoch": 0.10245687401986409, "grad_norm": 0.08306451886892319, "kl": 0.0716552734375, "learning_rate": 1.999712055165124e-05, "loss": 0.0067, "num_tokens": 172903691.0, "reward": 0.6121652126312256, "reward_std": 0.15384608693420887, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.3076340928673744, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 343 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6584821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 965.0045013427734, "completions/mean_terminated_length": 860.6990356445312, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "epoch": 0.10275558210738556, "grad_norm": 0.07556742429733276, "kl": 0.065673828125, "learning_rate": 1.999685284838938e-05, "loss": 0.0012, "num_tokens": 173406365.0, "reward": 0.6601562798023224, "reward_std": 0.167361319065094, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.3664930537343025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 344 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 911.5201416015625, "completions/mean_terminated_length": 821.9295349121094, "completions/min_length": 478.5, "completions/min_terminated_length": 478.5, "epoch": 0.10305429019490703, "grad_norm": 0.07969692349433899, "kl": 0.0732421875, "learning_rate": 1.9996573249755573e-05, "loss": 0.0107, "num_tokens": 173889782.0, "reward": 0.6328125447034836, "reward_std": 0.1143770469352603, "rewards/accuracy_reward/mean": 0.1339285704307258, "rewards/accuracy_reward/std": 0.2944314144551754, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 345 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 957.0469055175781, "completions/mean_terminated_length": 830.30810546875, "completions/min_length": 535.0, "completions/min_terminated_length": 535.0, "epoch": 0.1033529982824285, "grad_norm": 0.06915570795536041, "kl": 0.0689697265625, "learning_rate": 1.999628175608252e-05, "loss": 0.0044, "num_tokens": 174398107.0, "reward": 0.5820312798023224, "reward_std": 0.09688593680039048, "rewards/accuracy_reward/mean": 0.08482143003493547, "rewards/accuracy_reward/std": 0.2227826490998268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 346 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6830357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 970.6295013427734, "completions/mean_terminated_length": 860.3799133300781, "completions/min_length": 506.25, "completions/min_terminated_length": 506.25, "epoch": 0.10365170636994997, "grad_norm": 0.0735507383942604, "kl": 0.06817626953125, "learning_rate": 1.9995978367717063e-05, "loss": 0.0104, "num_tokens": 174902997.0, "reward": 0.6328125298023224, "reward_std": 0.08342665852978826, "rewards/accuracy_reward/mean": 0.13616071455180645, "rewards/accuracy_reward/std": 0.2853558510541916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 347 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 965.3259429931641, "completions/mean_terminated_length": 854.0769500732422, "completions/min_length": 596.25, "completions/min_terminated_length": 596.25, "epoch": 0.10395041445747144, "grad_norm": 0.06901726871728897, "kl": 0.07568359375, "learning_rate": 1.9995663085020215e-05, "loss": 0.006, "num_tokens": 175401671.0, "reward": 0.6088169813156128, "reward_std": 0.14031982980668545, "rewards/accuracy_reward/mean": 0.11160714598372579, "rewards/accuracy_reward/std": 0.29084303602576256, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 348 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6808035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 965.7723693847656, "completions/mean_terminated_length": 854.1877746582031, "completions/min_length": 455.25, "completions/min_terminated_length": 455.25, "epoch": 0.10424912254499291, "grad_norm": 0.07222647219896317, "kl": 0.07177734375, "learning_rate": 1.9995335908367132e-05, "loss": -0.0033, "num_tokens": 175906113.0, "reward": 0.6372768133878708, "reward_std": 0.12123831361532211, "rewards/accuracy_reward/mean": 0.14062499813735485, "rewards/accuracy_reward/std": 0.2764870971441269, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.023462072014808655, "step": 349 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7120535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.5, "completions/mean_length": 969.5045166015625, "completions/mean_terminated_length": 829.0920104980469, "completions/min_length": 533.25, "completions/min_terminated_length": 533.25, "epoch": 0.10454783063251437, "grad_norm": 0.10029619187116623, "kl": 0.0765380859375, "learning_rate": 1.9994996838147124e-05, "loss": 0.012, "num_tokens": 176419299.0, "reward": 0.7064732611179352, "reward_std": 0.1890929453074932, "rewards/accuracy_reward/mean": 0.23028273321688175, "rewards/accuracy_reward/std": 0.39205507934093475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 350 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 983.4152374267578, "completions/mean_terminated_length": 870.7580718994141, "completions/min_length": 528.75, "completions/min_terminated_length": 528.75, "epoch": 0.10484653872003584, "grad_norm": 0.09012504667043686, "kl": 0.0772705078125, "learning_rate": 1.9994645874763657e-05, "loss": -0.0011, "num_tokens": 176929469.0, "reward": 0.5412946566939354, "reward_std": 0.0909947669133544, "rewards/accuracy_reward/mean": 0.0558035708963871, "rewards/accuracy_reward/std": 0.19313222914934158, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 351 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7031250000000001, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 980.3482360839844, "completions/mean_terminated_length": 875.3993377685547, "completions/min_length": 570.5, "completions/min_terminated_length": 570.5, "epoch": 0.10514524680755731, "grad_norm": 0.0904926136136055, "kl": 0.0755615234375, "learning_rate": 1.9994283018634348e-05, "loss": 0.0066, "num_tokens": 177444681.0, "reward": 0.674107164144516, "reward_std": 0.12943676603026688, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.3736211508512497, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 352 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 970.7723693847656, "completions/mean_terminated_length": 864.1841735839844, "completions/min_length": 581.75, "completions/min_terminated_length": 581.75, "epoch": 0.10544395489507878, "grad_norm": 0.09794013947248459, "kl": 0.0758056640625, "learning_rate": 1.999390827019096e-05, "loss": 0.0081, "num_tokens": 177954147.0, "reward": 0.579241082072258, "reward_std": 0.11017661169171333, "rewards/accuracy_reward/mean": 0.09151785587891936, "rewards/accuracy_reward/std": 0.26755835488438606, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05265482235699892, "step": 353 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7388392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 984.0670166015625, "completions/mean_terminated_length": 868.2343902587891, "completions/min_length": 614.0, "completions/min_terminated_length": 614.0, "epoch": 0.10574266298260025, "grad_norm": 0.09086660295724869, "kl": 0.0823974609375, "learning_rate": 1.999352162987941e-05, "loss": 0.0065, "num_tokens": 178461041.0, "reward": 0.6383928805589676, "reward_std": 0.1397440806031227, "rewards/accuracy_reward/mean": 0.14955356903374195, "rewards/accuracy_reward/std": 0.34943923354148865, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 354 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6964285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 968.5937805175781, "completions/mean_terminated_length": 838.8587036132812, "completions/min_length": 567.0, "completions/min_terminated_length": 567.0, "epoch": 0.10604137107012172, "grad_norm": 0.08728675544261932, "kl": 0.078369140625, "learning_rate": 1.999312309815977e-05, "loss": 0.0031, "num_tokens": 178968443.0, "reward": 0.5714285969734192, "reward_std": 0.12393703311681747, "rewards/accuracy_reward/mean": 0.08258928591385484, "rewards/accuracy_reward/std": 0.2631937190890312, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392761349678, "rewards/tag_count_reward/std": 0.04965366888791323, "step": 355 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 970.0937805175781, "completions/mean_terminated_length": 867.8503570556641, "completions/min_length": 585.25, "completions/min_terminated_length": 585.25, "epoch": 0.1063400791576432, "grad_norm": 0.08736217021942139, "kl": 0.0750732421875, "learning_rate": 1.9992712675506253e-05, "loss": -0.0036, "num_tokens": 179473685.0, "reward": 0.6060267984867096, "reward_std": 0.13830003887414932, "rewards/accuracy_reward/mean": 0.11383928637951612, "rewards/accuracy_reward/std": 0.3043038807809353, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 356 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6852678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 980.7143249511719, "completions/mean_terminated_length": 887.7540740966797, "completions/min_length": 589.75, "completions/min_terminated_length": 589.75, "epoch": 0.10663878724516467, "grad_norm": 0.08877677470445633, "kl": 0.0765380859375, "learning_rate": 1.9992290362407232e-05, "loss": 0.0047, "num_tokens": 179980917.0, "reward": 0.5279018059372902, "reward_std": 0.08874704595655203, "rewards/accuracy_reward/mean": 0.03571428544819355, "rewards/accuracy_reward/std": 0.14853783324360847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 357 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6428571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 976.9286193847656, "completions/mean_terminated_length": 894.7723236083984, "completions/min_length": 612.75, "completions/min_terminated_length": 612.75, "epoch": 0.10693749533268614, "grad_norm": 0.09180416166782379, "kl": 0.07275390625, "learning_rate": 1.9991856159365214e-05, "loss": 0.0104, "num_tokens": 180493973.0, "reward": 0.6640625298023224, "reward_std": 0.1618828373029828, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.36790701001882553, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 358 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6138392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 961.1161193847656, "completions/mean_terminated_length": 862.5067901611328, "completions/min_length": 530.75, "completions/min_terminated_length": 530.75, "epoch": 0.10723620342020761, "grad_norm": 0.0856718122959137, "kl": 0.0714111328125, "learning_rate": 1.999141006689687e-05, "loss": 0.0051, "num_tokens": 180993561.0, "reward": 0.5859375298023224, "reward_std": 0.0989985961932689, "rewards/accuracy_reward/mean": 0.08928571548312902, "rewards/accuracy_reward/std": 0.19655770808458328, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517761349678, "rewards/tag_count_reward/std": 0.024942624382674694, "step": 359 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 952.4286193847656, "completions/mean_terminated_length": 840.9058990478516, "completions/min_length": 523.25, "completions/min_terminated_length": 523.25, "epoch": 0.10753491150772908, "grad_norm": 0.07400032877922058, "kl": 0.072021484375, "learning_rate": 1.9990952085533005e-05, "loss": 0.0097, "num_tokens": 181491737.0, "reward": 0.6054687798023224, "reward_std": 0.10387003049254417, "rewards/accuracy_reward/mean": 0.11495535541325808, "rewards/accuracy_reward/std": 0.30890539288520813, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 360 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 961.6339721679688, "completions/mean_terminated_length": 874.3356628417969, "completions/min_length": 457.5, "completions/min_terminated_length": 457.5, "epoch": 0.10783361959525053, "grad_norm": 0.07359503209590912, "kl": 0.0687255859375, "learning_rate": 1.999048221581858e-05, "loss": 0.0091, "num_tokens": 181992597.0, "reward": 0.6104910969734192, "reward_std": 0.1054292181506753, "rewards/accuracy_reward/mean": 0.11383928661234677, "rewards/accuracy_reward/std": 0.28356896713376045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 361 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 965.2768249511719, "completions/mean_terminated_length": 866.060302734375, "completions/min_length": 627.0, "completions/min_terminated_length": 627.0, "epoch": 0.108132327682772, "grad_norm": 0.08485594391822815, "kl": 0.0714111328125, "learning_rate": 1.9990000458312696e-05, "loss": 0.0048, "num_tokens": 182497233.0, "reward": 0.6679687798023224, "reward_std": 0.1814307849854231, "rewards/accuracy_reward/mean": 0.17187499813735485, "rewards/accuracy_reward/std": 0.3735724464058876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 362 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 932.310302734375, "completions/mean_terminated_length": 814.8820037841797, "completions/min_length": 447.5, "completions/min_terminated_length": 447.5, "epoch": 0.10843103577029348, "grad_norm": 0.07515507936477661, "kl": 0.0736083984375, "learning_rate": 1.9989506813588606e-05, "loss": 0.0074, "num_tokens": 182980572.0, "reward": 0.6523437798023224, "reward_std": 0.11787519231438637, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.3441788963973522, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 363 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 969.2366485595703, "completions/mean_terminated_length": 874.3583374023438, "completions/min_length": 541.0, "completions/min_terminated_length": 541.0, "epoch": 0.10872974385781495, "grad_norm": 0.07389691472053528, "kl": 0.0704345703125, "learning_rate": 1.9989001282233693e-05, "loss": 0.0096, "num_tokens": 183484982.0, "reward": 0.5435267984867096, "reward_std": 0.09853816591203213, "rewards/accuracy_reward/mean": 0.05357142980210483, "rewards/accuracy_reward/std": 0.20871136710047722, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03161557391285896, "step": 364 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 965.1629943847656, "completions/mean_terminated_length": 872.1188659667969, "completions/min_length": 568.0, "completions/min_terminated_length": 568.0, "epoch": 0.10902845194533642, "grad_norm": 0.08890101313591003, "kl": 0.07073974609375, "learning_rate": 1.998848386484951e-05, "loss": 0.0067, "num_tokens": 183988111.0, "reward": 0.6378348618745804, "reward_std": 0.1552256802096963, "rewards/accuracy_reward/mean": 0.14062499813735485, "rewards/accuracy_reward/std": 0.3437420427799225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.017556377220898867, "step": 365 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 962.0893249511719, "completions/mean_terminated_length": 873.5694274902344, "completions/min_length": 572.5, "completions/min_terminated_length": 572.5, "epoch": 0.10932716003285789, "grad_norm": 0.06559224426746368, "kl": 0.06549072265625, "learning_rate": 1.9987954562051724e-05, "loss": 0.0049, "num_tokens": 184493751.0, "reward": 0.595982164144516, "reward_std": 0.09456491190940142, "rewards/accuracy_reward/mean": 0.09821428265422583, "rewards/accuracy_reward/std": 0.24713555723428726, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 366 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7723214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 987.247802734375, "completions/mean_terminated_length": 871.2278289794922, "completions/min_length": 629.0, "completions/min_terminated_length": 629.0, "epoch": 0.10962586812037936, "grad_norm": 0.07493925839662552, "kl": 0.066650390625, "learning_rate": 1.998741337447017e-05, "loss": 0.0042, "num_tokens": 185006262.0, "reward": 0.6395089477300644, "reward_std": 0.14908828772604465, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.35199425369501114, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 367 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6941964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 975.8236999511719, "completions/mean_terminated_length": 867.7378387451172, "completions/min_length": 636.25, "completions/min_terminated_length": 636.25, "epoch": 0.10992457620790083, "grad_norm": 0.08328256011009216, "kl": 0.0654296875, "learning_rate": 1.9986860302748804e-05, "loss": 0.0104, "num_tokens": 185511735.0, "reward": 0.6143973618745804, "reward_std": 0.14675607532262802, "rewards/accuracy_reward/mean": 0.12053571362048388, "rewards/accuracy_reward/std": 0.30468397215008736, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 368 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7678571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 984.3170013427734, "completions/mean_terminated_length": 868.1149291992188, "completions/min_length": 584.25, "completions/min_terminated_length": 584.25, "epoch": 0.1102232842954223, "grad_norm": 0.08009056001901627, "kl": 0.0679931640625, "learning_rate": 1.9986295347545738e-05, "loss": 0.0043, "num_tokens": 186027925.0, "reward": 0.560825914144516, "reward_std": 0.09805220365524292, "rewards/accuracy_reward/mean": 0.0691964291036129, "rewards/accuracy_reward/std": 0.22368070855736732, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.04348720656707883, "step": 369 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7366071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 981.4152069091797, "completions/mean_terminated_length": 864.5213317871094, "completions/min_length": 568.0, "completions/min_terminated_length": 568.0, "epoch": 0.11052199238294377, "grad_norm": 0.08272796124219894, "kl": 0.07080078125, "learning_rate": 1.998571850953322e-05, "loss": 0.0108, "num_tokens": 186548783.0, "reward": 0.5496652126312256, "reward_std": 0.11283298581838608, "rewards/accuracy_reward/mean": 0.0558035708963871, "rewards/accuracy_reward/std": 0.21807417273521423, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 370 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 958.2812805175781, "completions/mean_terminated_length": 853.3779449462891, "completions/min_length": 563.0, "completions/min_terminated_length": 563.0, "epoch": 0.11082070047046524, "grad_norm": 0.08587808161973953, "kl": 0.06640625, "learning_rate": 1.9985129789397633e-05, "loss": 0.0013, "num_tokens": 187050013.0, "reward": 0.6456473469734192, "reward_std": 0.15845898538827896, "rewards/accuracy_reward/mean": 0.1495535741560161, "rewards/accuracy_reward/std": 0.3259423300623894, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937574505806, "rewards/tag_count_reward/std": 0.029367766808718443, "step": 371 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7165178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 987.7522735595703, "completions/mean_terminated_length": 897.8469848632812, "completions/min_length": 643.25, "completions/min_terminated_length": 643.25, "epoch": 0.1111194085579867, "grad_norm": 0.07484855502843857, "kl": 0.0672607421875, "learning_rate": 1.9984529187839504e-05, "loss": 0.0073, "num_tokens": 187559070.0, "reward": 0.584263414144516, "reward_std": 0.10077358270063996, "rewards/accuracy_reward/mean": 0.09151785913854837, "rewards/accuracy_reward/std": 0.23288775235414505, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.037192290648818016, "step": 372 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7120535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 975.1339721679688, "completions/mean_terminated_length": 871.0562591552734, "completions/min_length": 615.5, "completions/min_terminated_length": 615.5, "epoch": 0.11141811664550817, "grad_norm": 0.09288478642702103, "kl": 0.0684814453125, "learning_rate": 1.998391670557349e-05, "loss": 0.0071, "num_tokens": 188064474.0, "reward": 0.6858259290456772, "reward_std": 0.20744670182466507, "rewards/accuracy_reward/mean": 0.19642857275903225, "rewards/accuracy_reward/std": 0.38510455191135406, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 373 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7031249999999999, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 986.6205749511719, "completions/mean_terminated_length": 899.2766418457031, "completions/min_length": 547.5, "completions/min_terminated_length": 547.5, "epoch": 0.11171682473302964, "grad_norm": 0.07916299253702164, "kl": 0.06439208984375, "learning_rate": 1.9983292343328397e-05, "loss": 0.0062, "num_tokens": 188576640.0, "reward": 0.6501116305589676, "reward_std": 0.21085495501756668, "rewards/accuracy_reward/mean": 0.1517857164144516, "rewards/accuracy_reward/std": 0.3428531624376774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258992433548, "rewards/tag_count_reward/std": 0.017717084381729364, "step": 374 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 953.0603179931641, "completions/mean_terminated_length": 838.2001953125, "completions/min_length": 549.75, "completions/min_terminated_length": 549.75, "epoch": 0.11201553282055111, "grad_norm": 0.070492684841156, "kl": 0.072021484375, "learning_rate": 1.998265610184716e-05, "loss": 0.0003, "num_tokens": 189075307.0, "reward": 0.6406250447034836, "reward_std": 0.12034641578793526, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.2931986376643181, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 375 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 958.3594207763672, "completions/mean_terminated_length": 849.005126953125, "completions/min_length": 517.75, "completions/min_terminated_length": 517.75, "epoch": 0.11231424090807259, "grad_norm": 0.07904241234064102, "kl": 0.0679931640625, "learning_rate": 1.998200798188685e-05, "loss": 0.006, "num_tokens": 189572988.0, "reward": 0.7315848469734192, "reward_std": 0.15923793707042933, "rewards/accuracy_reward/mean": 0.23660714784637094, "rewards/accuracy_reward/std": 0.3662501238286495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 376 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6830357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 971.3438110351562, "completions/mean_terminated_length": 872.9011840820312, "completions/min_length": 652.75, "completions/min_terminated_length": 652.75, "epoch": 0.11261294899559406, "grad_norm": 0.09017835557460785, "kl": 0.0723876953125, "learning_rate": 1.998134798421867e-05, "loss": 0.0011, "num_tokens": 190080454.0, "reward": 0.636160746216774, "reward_std": 0.17842988297343254, "rewards/accuracy_reward/mean": 0.14285713899880648, "rewards/accuracy_reward/std": 0.3372587263584137, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03759844787418842, "step": 377 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6584821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 961.6339721679688, "completions/mean_terminated_length": 843.6349029541016, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "epoch": 0.11291165708311553, "grad_norm": 0.07135369628667831, "kl": 0.071533203125, "learning_rate": 1.9980676109627962e-05, "loss": 0.0052, "num_tokens": 190587506.0, "reward": 0.6445312798023224, "reward_std": 0.13690804690122604, "rewards/accuracy_reward/mean": 0.1473214253783226, "rewards/accuracy_reward/std": 0.21058687567710876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 378 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 978.0402221679688, "completions/mean_terminated_length": 868.6643981933594, "completions/min_length": 613.0, "completions/min_terminated_length": 613.0, "epoch": 0.113210365170637, "grad_norm": 0.06497974693775177, "kl": 0.0745849609375, "learning_rate": 1.99799923589142e-05, "loss": 0.0034, "num_tokens": 191101380.0, "reward": 0.6104910969734192, "reward_std": 0.1302151766140014, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.2585744895040989, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 379 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 971.6763763427734, "completions/mean_terminated_length": 867.5479888916016, "completions/min_length": 607.0, "completions/min_terminated_length": 607.0, "epoch": 0.11350907325815847, "grad_norm": 0.07251288741827011, "kl": 0.072021484375, "learning_rate": 1.9979296732890978e-05, "loss": 0.0107, "num_tokens": 191609171.0, "reward": 0.6395089626312256, "reward_std": 0.1237183678895235, "rewards/accuracy_reward/mean": 0.14285714458674192, "rewards/accuracy_reward/std": 0.33909496665000916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 380 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6584821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 971.6250610351562, "completions/mean_terminated_length": 874.0878753662109, "completions/min_length": 583.5, "completions/min_terminated_length": 583.5, "epoch": 0.11380778134567994, "grad_norm": 0.07820339500904083, "kl": 0.0745849609375, "learning_rate": 1.9978589232386036e-05, "loss": 0.0089, "num_tokens": 192117883.0, "reward": 0.6305803805589676, "reward_std": 0.1559248836711049, "rewards/accuracy_reward/mean": 0.13616071455180645, "rewards/accuracy_reward/std": 0.34370172023773193, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.030521792825311422, "step": 381 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 936.2812957763672, "completions/mean_terminated_length": 832.5697174072266, "completions/min_length": 507.5, "completions/min_terminated_length": 507.5, "epoch": 0.11410648943320141, "grad_norm": 0.08870731294155121, "kl": 0.0772705078125, "learning_rate": 1.9977869858241235e-05, "loss": 0.01, "num_tokens": 192607593.0, "reward": 0.6395089626312256, "reward_std": 0.15482164174318314, "rewards/accuracy_reward/mean": 0.14508928591385484, "rewards/accuracy_reward/std": 0.3267012722790241, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03141601476818323, "step": 382 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 939.5870819091797, "completions/mean_terminated_length": 802.6189575195312, "completions/min_length": 512.5, "completions/min_terminated_length": 512.5, "epoch": 0.11440519752072287, "grad_norm": 0.08159848302602768, "kl": 0.0740966796875, "learning_rate": 1.997713861131257e-05, "loss": 0.0117, "num_tokens": 193102928.0, "reward": 0.6880580633878708, "reward_std": 0.17591572925448418, "rewards/accuracy_reward/mean": 0.1964285704307258, "rewards/accuracy_reward/std": 0.3661172613501549, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 383 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6741071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 972.8549499511719, "completions/mean_terminated_length": 877.9281921386719, "completions/min_length": 637.25, "completions/min_terminated_length": 637.25, "epoch": 0.11470390560824434, "grad_norm": 0.087724469602108, "kl": 0.07080078125, "learning_rate": 1.997639549247016e-05, "loss": 0.0039, "num_tokens": 193608431.0, "reward": 0.7890625298023224, "reward_std": 0.19968442060053349, "rewards/accuracy_reward/mean": 0.2946428656578064, "rewards/accuracy_reward/std": 0.4193514995276928, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 384 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6696428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 966.0625305175781, "completions/mean_terminated_length": 850.2074890136719, "completions/min_length": 606.75, "completions/min_terminated_length": 606.75, "epoch": 0.11500261369576581, "grad_norm": 0.09423436969518661, "kl": 0.0732421875, "learning_rate": 1.9975640502598243e-05, "loss": 0.0069, "num_tokens": 194112091.0, "reward": 0.6964285969734192, "reward_std": 0.21679966989904642, "rewards/accuracy_reward/mean": 0.2053571380674839, "rewards/accuracy_reward/std": 0.40138213336467743, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.039923434145748615, "step": 385 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7767857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 989.1339721679688, "completions/mean_terminated_length": 869.4700469970703, "completions/min_length": 541.5, "completions/min_terminated_length": 541.5, "epoch": 0.11530132178328728, "grad_norm": 0.09425818175077438, "kl": 0.07568359375, "learning_rate": 1.99748736425952e-05, "loss": 0.0077, "num_tokens": 194627415.0, "reward": 0.5943080484867096, "reward_std": 0.11238851211965084, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.25744402408599854, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06465652491897345, "step": 386 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 956.5246124267578, "completions/mean_terminated_length": 843.9983825683594, "completions/min_length": 559.75, "completions/min_terminated_length": 559.75, "epoch": 0.11560002987080875, "grad_norm": 0.0935206413269043, "kl": 0.0787353515625, "learning_rate": 1.997409491337352e-05, "loss": 0.0079, "num_tokens": 195134610.0, "reward": 0.6255580484867096, "reward_std": 0.18203769996762276, "rewards/accuracy_reward/mean": 0.13616071082651615, "rewards/accuracy_reward/std": 0.34372011572122574, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932655245066, "step": 387 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 981.3527221679688, "completions/mean_terminated_length": 878.0606231689453, "completions/min_length": 675.25, "completions/min_terminated_length": 675.25, "epoch": 0.11589873795833022, "grad_norm": 0.07351963222026825, "kl": 0.0753173828125, "learning_rate": 1.9973304315859828e-05, "loss": 0.009, "num_tokens": 195647600.0, "reward": 0.6082589626312256, "reward_std": 0.15111559443175793, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.31018945574760437, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 388 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7120535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 979.8147888183594, "completions/mean_terminated_length": 872.5031127929688, "completions/min_length": 620.25, "completions/min_terminated_length": 620.25, "epoch": 0.1161974460458517, "grad_norm": 0.08718974888324738, "kl": 0.0675048828125, "learning_rate": 1.9972501850994857e-05, "loss": 0.0051, "num_tokens": 196159949.0, "reward": 0.7388393133878708, "reward_std": 0.163270965218544, "rewards/accuracy_reward/mean": 0.245535708963871, "rewards/accuracy_reward/std": 0.4243682697415352, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.032753214705735445, "step": 389 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7299107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 984.3594207763672, "completions/mean_terminated_length": 882.0077362060547, "completions/min_length": 582.25, "completions/min_terminated_length": 582.25, "epoch": 0.11649615413337316, "grad_norm": 0.09706147760152817, "kl": 0.06646728515625, "learning_rate": 1.997168751973348e-05, "loss": 0.0135, "num_tokens": 196666958.0, "reward": 0.7131696715950966, "reward_std": 0.19141335133463144, "rewards/accuracy_reward/mean": 0.2254464365541935, "rewards/accuracy_reward/std": 0.3230621889233589, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 390 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 986.8817443847656, "completions/mean_terminated_length": 874.2259979248047, "completions/min_length": 538.5, "completions/min_terminated_length": 538.5, "epoch": 0.11679486222089464, "grad_norm": 0.097767174243927, "kl": 0.0723876953125, "learning_rate": 1.9970861323044667e-05, "loss": 0.0059, "num_tokens": 197182185.0, "reward": 0.5781250298023224, "reward_std": 0.1706857718527317, "rewards/accuracy_reward/mean": 0.09412202145904303, "rewards/accuracy_reward/std": 0.28093067556619644, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05548384413123131, "step": 391 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7410714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 985.3058471679688, "completions/mean_terminated_length": 877.9216003417969, "completions/min_length": 584.0, "completions/min_terminated_length": 584.0, "epoch": 0.1170935703084161, "grad_norm": 0.08245641738176346, "kl": 0.0716552734375, "learning_rate": 1.9970023261911532e-05, "loss": 0.0069, "num_tokens": 197689362.0, "reward": 0.6702009290456772, "reward_std": 0.12854844518005848, "rewards/accuracy_reward/mean": 0.17410714412108064, "rewards/accuracy_reward/std": 0.3416639305651188, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 392 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7388392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 984.9464721679688, "completions/mean_terminated_length": 884.8158721923828, "completions/min_length": 664.0, "completions/min_terminated_length": 664.0, "epoch": 0.11739227839593756, "grad_norm": 0.08320289105176926, "kl": 0.06884765625, "learning_rate": 1.9969173337331283e-05, "loss": 0.0077, "num_tokens": 198200346.0, "reward": 0.6026785969734192, "reward_std": 0.14131325110793114, "rewards/accuracy_reward/mean": 0.10714285634458065, "rewards/accuracy_reward/std": 0.3018778786063194, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027692769188433886, "step": 393 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 965.8437957763672, "completions/mean_terminated_length": 841.5781402587891, "completions/min_length": 548.5, "completions/min_terminated_length": 548.5, "epoch": 0.11769098648345903, "grad_norm": 0.08033265173435211, "kl": 0.071044921875, "learning_rate": 1.996831155031526e-05, "loss": 0.0057, "num_tokens": 198710596.0, "reward": 0.7098214477300644, "reward_std": 0.12815717980265617, "rewards/accuracy_reward/mean": 0.2142857164144516, "rewards/accuracy_reward/std": 0.3943740874528885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.031776280142366886, "step": 394 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.734375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 979.1741485595703, "completions/mean_terminated_length": 874.7706756591797, "completions/min_length": 552.5, "completions/min_terminated_length": 552.5, "epoch": 0.1179896945709805, "grad_norm": 0.07229838520288467, "kl": 0.071533203125, "learning_rate": 1.9967437901888914e-05, "loss": 0.0025, "num_tokens": 199225090.0, "reward": 0.6763393133878708, "reward_std": 0.0978008322417736, "rewards/accuracy_reward/mean": 0.1785714291036129, "rewards/accuracy_reward/std": 0.379187636077404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 395 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 943.3839721679688, "completions/mean_terminated_length": 838.1576080322266, "completions/min_length": 599.0, "completions/min_terminated_length": 599.0, "epoch": 0.11828840265850198, "grad_norm": 0.0714658871293068, "kl": 0.0731201171875, "learning_rate": 1.9966552393091804e-05, "loss": 0.0011, "num_tokens": 199719022.0, "reward": 0.6316964626312256, "reward_std": 0.08441086765378714, "rewards/accuracy_reward/mean": 0.13392857555299997, "rewards/accuracy_reward/std": 0.3109425865113735, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 396 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 940.8505096435547, "completions/mean_terminated_length": 839.5441741943359, "completions/min_length": 520.25, "completions/min_terminated_length": 520.25, "epoch": 0.11858711074602345, "grad_norm": 0.06012814864516258, "kl": 0.0709228515625, "learning_rate": 1.9965655024977617e-05, "loss": 0.0089, "num_tokens": 200210091.0, "reward": 0.772879496216774, "reward_std": 0.11965414136648178, "rewards/accuracy_reward/mean": 0.2745535708963871, "rewards/accuracy_reward/std": 0.4272562190890312, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 397 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 956.9844360351562, "completions/mean_terminated_length": 849.7494506835938, "completions/min_length": 536.0, "completions/min_terminated_length": 536.0, "epoch": 0.11888581883354492, "grad_norm": 0.08918550610542297, "kl": 0.072021484375, "learning_rate": 1.9964745798614134e-05, "loss": 0.0027, "num_tokens": 200717076.0, "reward": 0.6422991305589676, "reward_std": 0.1546464692801237, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.33978698402643204, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.01845060009509325, "step": 398 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6830357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 972.8705749511719, "completions/mean_terminated_length": 863.1550445556641, "completions/min_length": 563.75, "completions/min_terminated_length": 563.75, "epoch": 0.11918452692106639, "grad_norm": 0.07814357429742813, "kl": 0.068359375, "learning_rate": 1.9963824715083255e-05, "loss": 0.0006, "num_tokens": 201220602.0, "reward": 0.6300223618745804, "reward_std": 0.11065002344548702, "rewards/accuracy_reward/mean": 0.13616071827709675, "rewards/accuracy_reward/std": 0.295893169939518, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.036084157414734364, "step": 399 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7433035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 990.794677734375, "completions/mean_terminated_length": 896.0384826660156, "completions/min_length": 683.0, "completions/min_terminated_length": 683.0, "epoch": 0.11948323500858786, "grad_norm": 0.08390458673238754, "kl": 0.07177734375, "learning_rate": 1.996289177548099e-05, "loss": 0.0019, "num_tokens": 201731950.0, "reward": 0.606026828289032, "reward_std": 0.1760451691225171, "rewards/accuracy_reward/mean": 0.10937500046566129, "rewards/accuracy_reward/std": 0.2883610241115093, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 400 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 991.8839721679688, "completions/mean_terminated_length": 888.3212890625, "completions/min_length": 593.5, "completions/min_terminated_length": 593.5, "epoch": 0.11978194309610933, "grad_norm": 0.07766752690076828, "kl": 0.065185546875, "learning_rate": 1.9961946980917457e-05, "loss": 0.0024, "num_tokens": 202251242.0, "reward": 0.6049107313156128, "reward_std": 0.14776582270860672, "rewards/accuracy_reward/mean": 0.11495535634458065, "rewards/accuracy_reward/std": 0.31979285925626755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.032210642006248236, "step": 401 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7477678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 994.1942596435547, "completions/mean_terminated_length": 910.6966400146484, "completions/min_length": 713.0, "completions/min_terminated_length": 713.0, "epoch": 0.1200806511836308, "grad_norm": 0.09689352661371231, "kl": 0.06884765625, "learning_rate": 1.9960990332516875e-05, "loss": 0.0048, "num_tokens": 202760417.0, "reward": 0.6501116305589676, "reward_std": 0.1681811809539795, "rewards/accuracy_reward/mean": 0.16071428824216127, "rewards/accuracy_reward/std": 0.3213140331208706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04909886047244072, "step": 402 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7812500000000001, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 997.1562957763672, "completions/mean_terminated_length": 902.162353515625, "completions/min_length": 665.75, "completions/min_terminated_length": 665.75, "epoch": 0.12037935927115227, "grad_norm": 0.09875739365816116, "kl": 0.0714111328125, "learning_rate": 1.996002183141757e-05, "loss": 0.0058, "num_tokens": 203283399.0, "reward": 0.6071428805589676, "reward_std": 0.15955160558223724, "rewards/accuracy_reward/mean": 0.11607142561115324, "rewards/accuracy_reward/std": 0.27557472698390484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04420433798804879, "step": 403 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7522321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 984.529052734375, "completions/mean_terminated_length": 860.3109893798828, "completions/min_length": 614.25, "completions/min_terminated_length": 614.25, "epoch": 0.12067806735867373, "grad_norm": 0.10385724902153015, "kl": 0.07080078125, "learning_rate": 1.9959041478771972e-05, "loss": 0.0016, "num_tokens": 203796948.0, "reward": 0.6668526977300644, "reward_std": 0.15082368813455105, "rewards/accuracy_reward/mean": 0.17410714481957257, "rewards/accuracy_reward/std": 0.3228226862847805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 404 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 995.1853179931641, "completions/mean_terminated_length": 884.5552062988281, "completions/min_length": 522.25, "completions/min_terminated_length": 522.25, "epoch": 0.1209767754461952, "grad_norm": 0.08108856528997421, "kl": 0.071044921875, "learning_rate": 1.9958049275746617e-05, "loss": 0.0039, "num_tokens": 204321255.0, "reward": 0.5731026977300644, "reward_std": 0.12848113104701042, "rewards/accuracy_reward/mean": 0.08035714202560484, "rewards/accuracy_reward/std": 0.24602900259196758, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.0412445142865181, "step": 405 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 962.9263610839844, "completions/mean_terminated_length": 873.8500213623047, "completions/min_length": 485.5, "completions/min_terminated_length": 485.5, "epoch": 0.12127548353371667, "grad_norm": 0.07838217169046402, "kl": 0.0767822265625, "learning_rate": 1.995704522352214e-05, "loss": 0.0049, "num_tokens": 204825030.0, "reward": 0.6333705633878708, "reward_std": 0.1255539134144783, "rewards/accuracy_reward/mean": 0.14546130690723658, "rewards/accuracy_reward/std": 0.3296462520956993, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 406 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 946.3415374755859, "completions/mean_terminated_length": 837.2484741210938, "completions/min_length": 440.25, "completions/min_terminated_length": 440.25, "epoch": 0.12157419162123814, "grad_norm": 0.060318417847156525, "kl": 0.0750732421875, "learning_rate": 1.9956029323293275e-05, "loss": 0.0022, "num_tokens": 205320591.0, "reward": 0.643973246216774, "reward_std": 0.09368257503956556, "rewards/accuracy_reward/mean": 0.14508928079158068, "rewards/accuracy_reward/std": 0.3348647207021713, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839253783226, "rewards/tag_count_reward/std": 0.008314208127558231, "step": 407 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 958.6272888183594, "completions/mean_terminated_length": 857.90283203125, "completions/min_length": 511.5, "completions/min_terminated_length": 511.5, "epoch": 0.12187289970875961, "grad_norm": 0.08653741329908371, "kl": 0.0740966796875, "learning_rate": 1.9955001576268848e-05, "loss": 0.0054, "num_tokens": 205821800.0, "reward": 0.6356027126312256, "reward_std": 0.2017345353960991, "rewards/accuracy_reward/mean": 0.13839285541325808, "rewards/accuracy_reward/std": 0.32826001197099686, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 408 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 935.5312957763672, "completions/mean_terminated_length": 841.7055358886719, "completions/min_length": 549.5, "completions/min_terminated_length": 549.5, "epoch": 0.12217160779628108, "grad_norm": 0.07631333917379379, "kl": 0.078369140625, "learning_rate": 1.9953961983671792e-05, "loss": 0.0064, "num_tokens": 206329158.0, "reward": 0.6690848618745804, "reward_std": 0.14093758538365364, "rewards/accuracy_reward/mean": 0.1718749962747097, "rewards/accuracy_reward/std": 0.36174921691417694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 409 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.734375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 986.872802734375, "completions/mean_terminated_length": 883.5791320800781, "completions/min_length": 560.5, "completions/min_terminated_length": 560.5, "epoch": 0.12247031588380256, "grad_norm": 0.0614829957485199, "kl": 0.077392578125, "learning_rate": 1.9952910546739128e-05, "loss": 0.0034, "num_tokens": 206843805.0, "reward": 0.5848214328289032, "reward_std": 0.07854388561099768, "rewards/accuracy_reward/mean": 0.08705356949940324, "rewards/accuracy_reward/std": 0.20525243878364563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678507566452, "rewards/tag_count_reward/std": 0.016628416255116463, "step": 410 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.671875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 964.4263610839844, "completions/mean_terminated_length": 847.5639495849609, "completions/min_length": 542.75, "completions/min_terminated_length": 542.75, "epoch": 0.12276902397132403, "grad_norm": 0.09047964215278625, "kl": 0.0731201171875, "learning_rate": 1.995184726672197e-05, "loss": 0.0031, "num_tokens": 207353692.0, "reward": 0.6277902126312256, "reward_std": 0.16608923487365246, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.3332001194357872, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 411 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 968.1161193847656, "completions/mean_terminated_length": 881.3067779541016, "completions/min_length": 582.5, "completions/min_terminated_length": 582.5, "epoch": 0.1230677320588455, "grad_norm": 0.0958617627620697, "kl": 0.0789794921875, "learning_rate": 1.9950772144885537e-05, "loss": 0.0074, "num_tokens": 207862384.0, "reward": 0.6021205484867096, "reward_std": 0.16466980054974556, "rewards/accuracy_reward/mean": 0.10937500186264515, "rewards/accuracy_reward/std": 0.31099630147218704, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03541599866002798, "step": 412 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 971.5178833007812, "completions/mean_terminated_length": 849.9607849121094, "completions/min_length": 513.25, "completions/min_terminated_length": 513.25, "epoch": 0.12336644014636697, "grad_norm": 0.099151611328125, "kl": 0.0733642578125, "learning_rate": 1.9949685182509117e-05, "loss": 0.0043, "num_tokens": 208377816.0, "reward": 0.7092634290456772, "reward_std": 0.1827289592474699, "rewards/accuracy_reward/mean": 0.2142857126891613, "rewards/accuracy_reward/std": 0.40130146592855453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.029007501434534788, "step": 413 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 977.4442443847656, "completions/mean_terminated_length": 874.0219116210938, "completions/min_length": 520.25, "completions/min_terminated_length": 520.25, "epoch": 0.12366514823388844, "grad_norm": 0.10137713700532913, "kl": 0.076171875, "learning_rate": 1.994858638088611e-05, "loss": 0.0122, "num_tokens": 208889071.0, "reward": 0.7276785969734192, "reward_std": 0.17881564423441887, "rewards/accuracy_reward/mean": 0.23437499813735485, "rewards/accuracy_reward/std": 0.3980696126818657, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 414 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6986607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 974.8103179931641, "completions/mean_terminated_length": 863.9120178222656, "completions/min_length": 493.25, "completions/min_terminated_length": 493.25, "epoch": 0.1239638563214099, "grad_norm": 0.09320608526468277, "kl": 0.073974609375, "learning_rate": 1.994747574132399e-05, "loss": 0.006, "num_tokens": 209402234.0, "reward": 0.6690848618745804, "reward_std": 0.15506064146757126, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.3676914758980274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.038913180120289326, "step": 415 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 1006.0513916015625, "completions/mean_terminated_length": 910.1951293945312, "completions/min_length": 729.75, "completions/min_terminated_length": 729.75, "epoch": 0.12426256440893137, "grad_norm": 0.07881221920251846, "kl": 0.0687255859375, "learning_rate": 1.9946353265144315e-05, "loss": 0.0062, "num_tokens": 209918177.0, "reward": 0.5725446790456772, "reward_std": 0.13578717038035393, "rewards/accuracy_reward/mean": 0.08258928544819355, "rewards/accuracy_reward/std": 0.27294185757637024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.046690347138792276, "step": 416 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 995.622802734375, "completions/mean_terminated_length": 863.6347503662109, "completions/min_length": 560.75, "completions/min_terminated_length": 560.75, "epoch": 0.12456127249645284, "grad_norm": 0.09647778421640396, "kl": 0.07568359375, "learning_rate": 1.9945218953682736e-05, "loss": 0.0052, "num_tokens": 210435128.0, "reward": 0.6556919813156128, "reward_std": 0.18751241639256477, "rewards/accuracy_reward/mean": 0.16517857555299997, "rewards/accuracy_reward/std": 0.34449438005685806, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 417 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.71875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 979.8170318603516, "completions/mean_terminated_length": 862.7120361328125, "completions/min_length": 599.25, "completions/min_terminated_length": 599.25, "epoch": 0.12485998058397431, "grad_norm": 0.09219985455274582, "kl": 0.0712890625, "learning_rate": 1.9944072808288984e-05, "loss": 0.012, "num_tokens": 210946342.0, "reward": 0.737723246216774, "reward_std": 0.23910926654934883, "rewards/accuracy_reward/mean": 0.2455357201397419, "rewards/accuracy_reward/std": 0.39853204041719437, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 418 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 997.9263916015625, "completions/mean_terminated_length": 902.2136840820312, "completions/min_length": 657.5, "completions/min_terminated_length": 657.5, "epoch": 0.1251586886714958, "grad_norm": 0.09068454056978226, "kl": 0.0723876953125, "learning_rate": 1.994291483032687e-05, "loss": 0.0052, "num_tokens": 211460933.0, "reward": 0.6824777126312256, "reward_std": 0.13454536721110344, "rewards/accuracy_reward/mean": 0.18750000558793545, "rewards/accuracy_reward/std": 0.36905550956726074, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.03359846491366625, "step": 419 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8169642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 998.6942291259766, "completions/mean_terminated_length": 885.2166442871094, "completions/min_length": 638.75, "completions/min_terminated_length": 638.75, "epoch": 0.12545739675901724, "grad_norm": 0.08594946563243866, "kl": 0.0701904296875, "learning_rate": 1.9941745021174284e-05, "loss": 0.0074, "num_tokens": 211982588.0, "reward": 0.6551339626312256, "reward_std": 0.17448382079601288, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.3528479263186455, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 420 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7388392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 983.5736999511719, "completions/mean_terminated_length": 874.1923370361328, "completions/min_length": 606.5, "completions/min_terminated_length": 606.5, "epoch": 0.1257561048465387, "grad_norm": 0.09665278345346451, "kl": 0.06787109375, "learning_rate": 1.9940563382223196e-05, "loss": 0.0114, "num_tokens": 212491901.0, "reward": 0.7723214626312256, "reward_std": 0.2486233375966549, "rewards/accuracy_reward/mean": 0.2767857126891613, "rewards/accuracy_reward/std": 0.43166791647672653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 421 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7455357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 983.6205749511719, "completions/mean_terminated_length": 864.0897827148438, "completions/min_length": 568.0, "completions/min_terminated_length": 568.0, "epoch": 0.12605481293406018, "grad_norm": 0.08376392722129822, "kl": 0.0703125, "learning_rate": 1.993936991487966e-05, "loss": 0.0079, "num_tokens": 213004147.0, "reward": 0.6997768133878708, "reward_std": 0.18060839269310236, "rewards/accuracy_reward/mean": 0.20312499860301614, "rewards/accuracy_reward/std": 0.35816915705800056, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 422 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 974.0580902099609, "completions/mean_terminated_length": 859.2101898193359, "completions/min_length": 498.75, "completions/min_terminated_length": 498.75, "epoch": 0.12635352102158165, "grad_norm": 0.07958398014307022, "kl": 0.0675048828125, "learning_rate": 1.993816462056378e-05, "loss": -0.0007, "num_tokens": 213508285.0, "reward": 0.7059152126312256, "reward_std": 0.18673680163919926, "rewards/accuracy_reward/mean": 0.20982142630964518, "rewards/accuracy_reward/std": 0.3742355778813362, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 423 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.78125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 992.6674652099609, "completions/mean_terminated_length": 878.4199829101562, "completions/min_length": 614.25, "completions/min_terminated_length": 614.25, "epoch": 0.12665222910910312, "grad_norm": 0.07549910992383957, "kl": 0.070556640625, "learning_rate": 1.9936947500709768e-05, "loss": 0.0018, "num_tokens": 214026904.0, "reward": 0.613839328289032, "reward_std": 0.11719677597284317, "rewards/accuracy_reward/mean": 0.11830357182770967, "rewards/accuracy_reward/std": 0.3163430690765381, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027185317594558, "step": 424 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7901785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 995.7053985595703, "completions/mean_terminated_length": 899.3601989746094, "completions/min_length": 622.0, "completions/min_terminated_length": 622.0, "epoch": 0.1269509371966246, "grad_norm": 0.07875382900238037, "kl": 0.0743408203125, "learning_rate": 1.9935718556765878e-05, "loss": 0.0032, "num_tokens": 214560372.0, "reward": 0.597098246216774, "reward_std": 0.10771467909216881, "rewards/accuracy_reward/mean": 0.10044642840512097, "rewards/accuracy_reward/std": 0.27657478861510754, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 425 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 989.9821929931641, "completions/mean_terminated_length": 894.6482849121094, "completions/min_length": 676.5, "completions/min_terminated_length": 676.5, "epoch": 0.12724964528414606, "grad_norm": 0.07376787811517715, "kl": 0.06671142578125, "learning_rate": 1.9934477790194445e-05, "loss": 0.0024, "num_tokens": 215073084.0, "reward": 0.6986607611179352, "reward_std": 0.1493135429918766, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.3816208206117153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 426 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 998.0714721679688, "completions/mean_terminated_length": 894.9770965576172, "completions/min_length": 630.75, "completions/min_terminated_length": 630.75, "epoch": 0.12754835337166753, "grad_norm": 0.08204950392246246, "kl": 0.072509765625, "learning_rate": 1.993322520247188e-05, "loss": 0.0025, "num_tokens": 215590508.0, "reward": 0.7165178954601288, "reward_std": 0.18534472212195396, "rewards/accuracy_reward/mean": 0.2209821455180645, "rewards/accuracy_reward/std": 0.39866795390844345, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 427 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7254464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 984.7009429931641, "completions/mean_terminated_length": 882.5213165283203, "completions/min_length": 567.25, "completions/min_terminated_length": 567.25, "epoch": 0.127847061459189, "grad_norm": 0.08049791306257248, "kl": 0.0736083984375, "learning_rate": 1.9931960795088637e-05, "loss": 0.007, "num_tokens": 216107878.0, "reward": 0.7064732611179352, "reward_std": 0.17548758909106255, "rewards/accuracy_reward/mean": 0.217261902987957, "rewards/accuracy_reward/std": 0.40631408244371414, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 428 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.78125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 986.5692443847656, "completions/mean_terminated_length": 860.1373596191406, "completions/min_length": 650.0, "completions/min_terminated_length": 650.0, "epoch": 0.12814576954671048, "grad_norm": 0.08662053197622299, "kl": 0.07049560546875, "learning_rate": 1.9930684569549265e-05, "loss": 0.0062, "num_tokens": 216626021.0, "reward": 0.6244419813156128, "reward_std": 0.14762836135923862, "rewards/accuracy_reward/mean": 0.1316964295692742, "rewards/accuracy_reward/std": 0.2948443293571472, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.042059858329594135, "step": 429 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 1008.1652374267578, "completions/mean_terminated_length": 927.6101379394531, "completions/min_length": 724.0, "completions/min_terminated_length": 724.0, "epoch": 0.12844447763423195, "grad_norm": 0.0810612142086029, "kl": 0.0738525390625, "learning_rate": 1.992939652737235e-05, "loss": 0.0048, "num_tokens": 217148303.0, "reward": 0.6143973469734192, "reward_std": 0.1256450517103076, "rewards/accuracy_reward/mean": 0.12276785587891936, "rewards/accuracy_reward/std": 0.29414090886712074, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.037647420540452003, "step": 430 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7745535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 991.2321929931641, "completions/mean_terminated_length": 888.1594390869141, "completions/min_length": 713.5, "completions/min_terminated_length": 713.5, "epoch": 0.12874318572175342, "grad_norm": 0.09447035938501358, "kl": 0.0692138671875, "learning_rate": 1.9928096670090552e-05, "loss": 0.0045, "num_tokens": 217656439.0, "reward": 0.6383928805589676, "reward_std": 0.1161548551172018, "rewards/accuracy_reward/mean": 0.15215773694217205, "rewards/accuracy_reward/std": 0.34851741045713425, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 431 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 1010.700927734375, "completions/mean_terminated_length": 920.4134674072266, "completions/min_length": 805.25, "completions/min_terminated_length": 805.25, "epoch": 0.1290418938092749, "grad_norm": 0.08307254314422607, "kl": 0.073974609375, "learning_rate": 1.9926784999250585e-05, "loss": 0.0029, "num_tokens": 218174209.0, "reward": 0.5786830633878708, "reward_std": 0.14629177609458566, "rewards/accuracy_reward/mean": 0.0848214291036129, "rewards/accuracy_reward/std": 0.2231709435582161, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 432 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 1007.0335235595703, "completions/mean_terminated_length": 932.1491241455078, "completions/min_length": 775.25, "completions/min_terminated_length": 775.25, "epoch": 0.12934060189679636, "grad_norm": 0.07797371596097946, "kl": 0.0711669921875, "learning_rate": 1.9925461516413224e-05, "loss": 0.0057, "num_tokens": 218696848.0, "reward": 0.5731026977300644, "reward_std": 0.12186668394133449, "rewards/accuracy_reward/mean": 0.08035714365541935, "rewards/accuracy_reward/std": 0.1756807565689087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 433 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 999.2455902099609, "completions/mean_terminated_length": 837.1517791748047, "completions/min_length": 583.75, "completions/min_terminated_length": 583.75, "epoch": 0.12963930998431783, "grad_norm": 0.07226742058992386, "kl": 0.072998046875, "learning_rate": 1.9924126223153292e-05, "loss": 0.0049, "num_tokens": 219215646.0, "reward": 0.6367187798023224, "reward_std": 0.09405616484582424, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.3476935848593712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 434 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8169642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 991.8973693847656, "completions/mean_terminated_length": 868.7335052490234, "completions/min_length": 620.0, "completions/min_terminated_length": 620.0, "epoch": 0.1299380180718393, "grad_norm": 0.07010232657194138, "kl": 0.0706787109375, "learning_rate": 1.992277912105967e-05, "loss": 0.0, "num_tokens": 219731584.0, "reward": 0.6238839477300644, "reward_std": 0.09502753335982561, "rewards/accuracy_reward/mean": 0.12500000186264515, "rewards/accuracy_reward/std": 0.32440662384033203, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 435 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 1003.7634429931641, "completions/mean_terminated_length": 890.6752624511719, "completions/min_length": 624.0, "completions/min_terminated_length": 624.0, "epoch": 0.13023672615936077, "grad_norm": 0.062228769063949585, "kl": 0.0718994140625, "learning_rate": 1.99214202117353e-05, "loss": 0.0001, "num_tokens": 220252822.0, "reward": 0.601004496216774, "reward_std": 0.08164609083905816, "rewards/accuracy_reward/mean": 0.10267857275903225, "rewards/accuracy_reward/std": 0.25455179810523987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 436 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.765625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 989.7857666015625, "completions/mean_terminated_length": 884.0199432373047, "completions/min_length": 622.5, "completions/min_terminated_length": 622.5, "epoch": 0.13053543424688224, "grad_norm": 0.06294349581003189, "kl": 0.072998046875, "learning_rate": 1.9920049496797153e-05, "loss": 0.0048, "num_tokens": 220764678.0, "reward": 0.5887276977300644, "reward_std": 0.11218438483774662, "rewards/accuracy_reward/mean": 0.09151785913854837, "rewards/accuracy_reward/std": 0.2808224558830261, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 437 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8013392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 1000.4710083007812, "completions/mean_terminated_length": 897.9371490478516, "completions/min_length": 636.75, "completions/min_terminated_length": 636.75, "epoch": 0.1308341423344037, "grad_norm": 0.08437301963567734, "kl": 0.0740966796875, "learning_rate": 1.991866697787626e-05, "loss": 0.005, "num_tokens": 221286777.0, "reward": 0.7539062798023224, "reward_std": 0.1967705562710762, "rewards/accuracy_reward/mean": 0.2566964253783226, "rewards/accuracy_reward/std": 0.42775124311447144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 438 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8348214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 1004.4353179931641, "completions/mean_terminated_length": 893.4412841796875, "completions/min_length": 582.0, "completions/min_terminated_length": 582.0, "epoch": 0.13113285042192518, "grad_norm": 0.06522975116968155, "kl": 0.075927734375, "learning_rate": 1.9917272656617704e-05, "loss": 0.001, "num_tokens": 221806028.0, "reward": 0.623325914144516, "reward_std": 0.10779064521193504, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.32749801129102707, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 439 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 987.7790679931641, "completions/mean_terminated_length": 881.5976104736328, "completions/min_length": 640.5, "completions/min_terminated_length": 640.5, "epoch": 0.13143155850944666, "grad_norm": 0.09189768880605698, "kl": 0.077880859375, "learning_rate": 1.99158665346806e-05, "loss": 0.0042, "num_tokens": 222327721.0, "reward": 0.6450893133878708, "reward_std": 0.14716415107250214, "rewards/accuracy_reward/mean": 0.16034225979819894, "rewards/accuracy_reward/std": 0.3378181643784046, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.039929782040417194, "step": 440 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 1009.8326263427734, "completions/mean_terminated_length": 926.9187469482422, "completions/min_length": 661.75, "completions/min_terminated_length": 661.75, "epoch": 0.13173026659696813, "grad_norm": 0.0865616649389267, "kl": 0.0762939453125, "learning_rate": 1.9914448613738107e-05, "loss": 0.0031, "num_tokens": 222847694.0, "reward": 0.569754496216774, "reward_std": 0.12623037211596966, "rewards/accuracy_reward/mean": 0.08035714132711291, "rewards/accuracy_reward/std": 0.26159490272402763, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 441 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8303571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 1002.0580749511719, "completions/mean_terminated_length": 899.2113494873047, "completions/min_length": 680.5, "completions/min_terminated_length": 680.5, "epoch": 0.13202897468448957, "grad_norm": 0.08236672729253769, "kl": 0.08154296875, "learning_rate": 1.9913018895477424e-05, "loss": 0.0039, "num_tokens": 223369416.0, "reward": 0.589285746216774, "reward_std": 0.11631641164422035, "rewards/accuracy_reward/mean": 0.09375000139698386, "rewards/accuracy_reward/std": 0.2256307601928711, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 442 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8325892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 999.9710388183594, "completions/mean_terminated_length": 884.9258728027344, "completions/min_length": 583.75, "completions/min_terminated_length": 583.75, "epoch": 0.13232768277201104, "grad_norm": 0.07621829956769943, "kl": 0.077880859375, "learning_rate": 1.99115773815998e-05, "loss": 0.0057, "num_tokens": 223894443.0, "reward": 0.6082589626312256, "reward_std": 0.12607190874405205, "rewards/accuracy_reward/mean": 0.11383928544819355, "rewards/accuracy_reward/std": 0.2674085795879364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 443 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 1015.6986999511719, "completions/mean_terminated_length": 931.3405456542969, "completions/min_length": 799.75, "completions/min_terminated_length": 799.75, "epoch": 0.1326263908595325, "grad_norm": 0.10424479842185974, "kl": 0.080810546875, "learning_rate": 1.9910124073820493e-05, "loss": 0.0054, "num_tokens": 224435284.0, "reward": 0.5731027126312256, "reward_std": 0.1367894597351551, "rewards/accuracy_reward/mean": 0.08705357229337096, "rewards/accuracy_reward/std": 0.2629600875079632, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.05592046119272709, "step": 444 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.890625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 1011.9263916015625, "completions/mean_terminated_length": 922.9273071289062, "completions/min_length": 716.0, "completions/min_terminated_length": 716.0, "epoch": 0.13292509894705398, "grad_norm": 0.08827546238899231, "kl": 0.070556640625, "learning_rate": 1.9908658973868823e-05, "loss": 0.0046, "num_tokens": 224962195.0, "reward": 0.6199777126312256, "reward_std": 0.13565723225474358, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.3223213702440262, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 445 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8616071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 1008.6183471679688, "completions/mean_terminated_length": 911.0038757324219, "completions/min_length": 743.75, "completions/min_terminated_length": 743.75, "epoch": 0.13322380703457545, "grad_norm": 0.08773555606603622, "kl": 0.0736083984375, "learning_rate": 1.9907182083488127e-05, "loss": 0.0052, "num_tokens": 225488072.0, "reward": 0.6752232536673546, "reward_std": 0.16396511066704988, "rewards/accuracy_reward/mean": 0.1852678544819355, "rewards/accuracy_reward/std": 0.3201136812567711, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.045422971714287996, "step": 446 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 1010.7768402099609, "completions/mean_terminated_length": 935.0134735107422, "completions/min_length": 789.25, "completions/min_terminated_length": 789.25, "epoch": 0.13352251512209692, "grad_norm": 0.08497801423072815, "kl": 0.0777587890625, "learning_rate": 1.990569340443577e-05, "loss": 0.0034, "num_tokens": 226010980.0, "reward": 0.6322544813156128, "reward_std": 0.17653464153409004, "rewards/accuracy_reward/mean": 0.13839285681024194, "rewards/accuracy_reward/std": 0.32118216529488564, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 447 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 1006.5804138183594, "completions/mean_terminated_length": 906.78125, "completions/min_length": 701.25, "completions/min_terminated_length": 701.25, "epoch": 0.1338212232096184, "grad_norm": 0.09578927606344223, "kl": 0.0797119140625, "learning_rate": 1.9904192938483157e-05, "loss": 0.0026, "num_tokens": 226531144.0, "reward": 0.5820312723517418, "reward_std": 0.1104016259778291, "rewards/accuracy_reward/mean": 0.0892857126891613, "rewards/accuracy_reward/std": 0.19199995696544647, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03406794602051377, "step": 448 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8549107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 1002.8549346923828, "completions/mean_terminated_length": 869.6465606689453, "completions/min_length": 533.75, "completions/min_terminated_length": 533.75, "epoch": 0.13411993129713987, "grad_norm": 0.067951999604702, "kl": 0.079833984375, "learning_rate": 1.9902680687415704e-05, "loss": 0.0057, "num_tokens": 227052199.0, "reward": 0.6322544813156128, "reward_std": 0.09246949572116137, "rewards/accuracy_reward/mean": 0.1361607164144516, "rewards/accuracy_reward/std": 0.34432777762413025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 449 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 1007.0111999511719, "completions/mean_terminated_length": 908.2973022460938, "completions/min_length": 637.25, "completions/min_terminated_length": 637.25, "epoch": 0.13441863938466134, "grad_norm": 0.06462069600820541, "kl": 0.0833740234375, "learning_rate": 1.990115665303286e-05, "loss": 0.0031, "num_tokens": 227570044.0, "reward": 0.5931919813156128, "reward_std": 0.0580559060908854, "rewards/accuracy_reward/mean": 0.0959821417927742, "rewards/accuracy_reward/std": 0.19426626712083817, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 450 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 1009.8795013427734, "completions/mean_terminated_length": 901.5313873291016, "completions/min_length": 532.75, "completions/min_terminated_length": 532.75, "epoch": 0.1347173474721828, "grad_norm": 0.06646958738565445, "kl": 0.07421875, "learning_rate": 1.989962083714808e-05, "loss": -0.0046, "num_tokens": 228088726.0, "reward": 0.709263414144516, "reward_std": 0.12430310621857643, "rewards/accuracy_reward/mean": 0.2142857164144516, "rewards/accuracy_reward/std": 0.3824021965265274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03751415526494384, "step": 451 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8415178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 1006.0670013427734, "completions/mean_terminated_length": 914.2960357666016, "completions/min_length": 693.5, "completions/min_terminated_length": 693.5, "epoch": 0.13501605555970428, "grad_norm": 0.0756043866276741, "kl": 0.0745849609375, "learning_rate": 1.989807324158886e-05, "loss": 0.0037, "num_tokens": 228608436.0, "reward": 0.7421875447034836, "reward_std": 0.1977187693119049, "rewards/accuracy_reward/mean": 0.24776785308495164, "rewards/accuracy_reward/std": 0.37444768473505974, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.02578705921769142, "step": 452 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 998.6562805175781, "completions/mean_terminated_length": 884.3258972167969, "completions/min_length": 638.25, "completions/min_terminated_length": 638.25, "epoch": 0.13531476364722575, "grad_norm": 0.09319688379764557, "kl": 0.08544921875, "learning_rate": 1.9896513868196706e-05, "loss": 0.0043, "num_tokens": 229130442.0, "reward": 0.5931919813156128, "reward_std": 0.1328178495168686, "rewards/accuracy_reward/mean": 0.10267856903374195, "rewards/accuracy_reward/std": 0.30326419323682785, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.044667141512036324, "step": 453 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 1009.2299499511719, "completions/mean_terminated_length": 903.1062164306641, "completions/min_length": 734.25, "completions/min_terminated_length": 734.25, "epoch": 0.13561347173474722, "grad_norm": 0.0701274424791336, "kl": 0.0816650390625, "learning_rate": 1.989494271882712e-05, "loss": 0.0045, "num_tokens": 229652577.0, "reward": 0.7165178954601288, "reward_std": 0.0888565662316978, "rewards/accuracy_reward/mean": 0.2187499925494194, "rewards/accuracy_reward/std": 0.3416772708296776, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 454 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 1014.0312805175781, "completions/mean_terminated_length": 950.4047698974609, "completions/min_length": 811.0, "completions/min_terminated_length": 811.0, "epoch": 0.1359121798222687, "grad_norm": 0.0730767473578453, "kl": 0.075439453125, "learning_rate": 1.9893359795349646e-05, "loss": 0.0022, "num_tokens": 230173071.0, "reward": 0.651785746216774, "reward_std": 0.1387872500345111, "rewards/accuracy_reward/mean": 0.1584821380674839, "rewards/accuracy_reward/std": 0.29709430783987045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 455 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 1001.6361999511719, "completions/mean_terminated_length": 887.4610443115234, "completions/min_length": 678.0, "completions/min_terminated_length": 678.0, "epoch": 0.13621088790979016, "grad_norm": 0.07603712379932404, "kl": 0.0772705078125, "learning_rate": 1.989176509964781e-05, "loss": 0.0048, "num_tokens": 230698668.0, "reward": 0.650669664144516, "reward_std": 0.12383191287517548, "rewards/accuracy_reward/mean": 0.15401785960420966, "rewards/accuracy_reward/std": 0.302933756262064, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517761349678, "rewards/tag_count_reward/std": 0.024942624382674694, "step": 456 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.75, "completions/mean_length": 1011.2321929931641, "completions/mean_terminated_length": 932.5013122558594, "completions/min_length": 806.5, "completions/min_terminated_length": 806.5, "epoch": 0.13650959599731163, "grad_norm": 0.0934717208147049, "kl": 0.0802001953125, "learning_rate": 1.989015863361917e-05, "loss": 0.0033, "num_tokens": 231231348.0, "reward": 0.6395089626312256, "reward_std": 0.20286840945482254, "rewards/accuracy_reward/mean": 0.14955357369035482, "rewards/accuracy_reward/std": 0.3451671376824379, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 457 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8303571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 1005.1585235595703, "completions/mean_terminated_length": 899.8773193359375, "completions/min_length": 560.0, "completions/min_terminated_length": 560.0, "epoch": 0.1368083040848331, "grad_norm": 0.07207787781953812, "kl": 0.0758056640625, "learning_rate": 1.988854039917527e-05, "loss": 0.0015, "num_tokens": 231761035.0, "reward": 0.6456473469734192, "reward_std": 0.13178234361112118, "rewards/accuracy_reward/mean": 0.15178571082651615, "rewards/accuracy_reward/std": 0.3572196513414383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 458 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 1020.0179138183594, "completions/mean_terminated_length": 926.3750152587891, "completions/min_length": 819.5, "completions/min_terminated_length": 819.5, "epoch": 0.13710701217235458, "grad_norm": 0.07754320651292801, "kl": 0.0750732421875, "learning_rate": 1.9886910398241673e-05, "loss": 0.0034, "num_tokens": 232280083.0, "reward": 0.6026786118745804, "reward_std": 0.15317491628229618, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.3036716356873512, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 459 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 1009.2366638183594, "completions/mean_terminated_length": 912.5150299072266, "completions/min_length": 676.5, "completions/min_terminated_length": 676.5, "epoch": 0.13740572025987605, "grad_norm": 0.088678739964962, "kl": 0.073974609375, "learning_rate": 1.9885268632757935e-05, "loss": 0.0053, "num_tokens": 232806909.0, "reward": 0.7120535969734192, "reward_std": 0.15615735948085785, "rewards/accuracy_reward/mean": 0.22544642724096775, "rewards/accuracy_reward/std": 0.41015487909317017, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.055819165892899036, "step": 460 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 1007.4688110351562, "completions/mean_terminated_length": 910.1159210205078, "completions/min_length": 702.75, "completions/min_terminated_length": 702.75, "epoch": 0.13770442834739752, "grad_norm": 0.08384883403778076, "kl": 0.0804443359375, "learning_rate": 1.988361510467761e-05, "loss": -0.0015, "num_tokens": 233333919.0, "reward": 0.6713169813156128, "reward_std": 0.17161208391189575, "rewards/accuracy_reward/mean": 0.18080356949940324, "rewards/accuracy_reward/std": 0.35193387046456337, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983869820833, "step": 461 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 1011.5915679931641, "completions/mean_terminated_length": 903.3562469482422, "completions/min_length": 708.0, "completions/min_terminated_length": 708.0, "epoch": 0.138003136434919, "grad_norm": 0.10087449848651886, "kl": 0.0723876953125, "learning_rate": 1.988194981596825e-05, "loss": 0.0049, "num_tokens": 233856696.0, "reward": 0.6808035969734192, "reward_std": 0.18425939604640007, "rewards/accuracy_reward/mean": 0.19866071082651615, "rewards/accuracy_reward/std": 0.37526971846818924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428507566452, "rewards/tag_count_reward/std": 0.06228448450565338, "step": 462 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.0, "completions/mean_length": 1017.8638763427734, "completions/mean_terminated_length": 909.4072113037109, "completions/min_length": 784.0, "completions/min_terminated_length": 784.0, "epoch": 0.13830184452244043, "grad_norm": 0.08921059966087341, "kl": 0.08056640625, "learning_rate": 1.9880272768611398e-05, "loss": 0.0023, "num_tokens": 234384747.0, "reward": 0.6043527126312256, "reward_std": 0.12589336186647415, "rewards/accuracy_reward/mean": 0.11383928405120969, "rewards/accuracy_reward/std": 0.29805218055844307, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 463 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 997.013427734375, "completions/mean_terminated_length": 876.6461029052734, "completions/min_length": 611.5, "completions/min_terminated_length": 611.5, "epoch": 0.1386005526099619, "grad_norm": 0.0869605764746666, "kl": 0.074951171875, "learning_rate": 1.98785839646026e-05, "loss": 0.006, "num_tokens": 234901345.0, "reward": 0.7154018133878708, "reward_std": 0.18663312681019306, "rewards/accuracy_reward/mean": 0.2209821450524032, "rewards/accuracy_reward/std": 0.374474436044693, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.03299933345988393, "step": 464 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.796875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 1000.9531707763672, "completions/mean_terminated_length": 902.011474609375, "completions/min_length": 646.5, "completions/min_terminated_length": 646.5, "epoch": 0.13889926069748337, "grad_norm": 0.0818309560418129, "kl": 0.081298828125, "learning_rate": 1.9876883405951378e-05, "loss": 0.0051, "num_tokens": 235432028.0, "reward": 0.711495578289032, "reward_std": 0.12658143043518066, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.3942866437137127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 465 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 994.3839874267578, "completions/mean_terminated_length": 895.2809906005859, "completions/min_length": 645.0, "completions/min_terminated_length": 645.0, "epoch": 0.13919796878500484, "grad_norm": 0.06965586543083191, "kl": 0.0782470703125, "learning_rate": 1.9875171094681248e-05, "loss": 0.0023, "num_tokens": 235958584.0, "reward": 0.5864955633878708, "reward_std": 0.13137000985443592, "rewards/accuracy_reward/mean": 0.08705357369035482, "rewards/accuracy_reward/std": 0.269259974360466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 466 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 1002.8839721679688, "completions/mean_terminated_length": 910.7525787353516, "completions/min_length": 737.25, "completions/min_terminated_length": 737.25, "epoch": 0.13949667687252632, "grad_norm": 0.06447212398052216, "kl": 0.080078125, "learning_rate": 1.9873447032829713e-05, "loss": 0.0036, "num_tokens": 236480340.0, "reward": 0.5864955633878708, "reward_std": 0.1042042151093483, "rewards/accuracy_reward/mean": 0.08705356903374195, "rewards/accuracy_reward/std": 0.24069952964782715, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 467 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 1002.2679138183594, "completions/mean_terminated_length": 910.0582427978516, "completions/min_length": 629.5, "completions/min_terminated_length": 629.5, "epoch": 0.13979538496004779, "grad_norm": 0.07696867734193802, "kl": 0.0802001953125, "learning_rate": 1.987171122244825e-05, "loss": 0.0011, "num_tokens": 237008476.0, "reward": 0.6183035969734192, "reward_std": 0.1286439262330532, "rewards/accuracy_reward/mean": 0.12053571501746774, "rewards/accuracy_reward/std": 0.30548960343003273, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 468 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.84375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 1006.5178985595703, "completions/mean_terminated_length": 916.3119964599609, "completions/min_length": 616.25, "completions/min_terminated_length": 616.25, "epoch": 0.14009409304756926, "grad_norm": 0.07545777410268784, "kl": 0.077392578125, "learning_rate": 1.9869963665602322e-05, "loss": 0.0033, "num_tokens": 237537188.0, "reward": 0.723214328289032, "reward_std": 0.14348457613959908, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.31776101142168045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 469 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.890625, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.25, "completions/mean_length": 1012.1361999511719, "completions/mean_terminated_length": 897.4514007568359, "completions/min_length": 767.75, "completions/min_terminated_length": 767.75, "epoch": 0.14039280113509073, "grad_norm": 0.08344603329896927, "kl": 0.083984375, "learning_rate": 1.9868204364371363e-05, "loss": 0.0013, "num_tokens": 238066577.0, "reward": 0.7243303954601288, "reward_std": 0.12080210447311401, "rewards/accuracy_reward/mean": 0.2380952388048172, "rewards/accuracy_reward/std": 0.4259032756090164, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 470 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 1003.4241485595703, "completions/mean_terminated_length": 911.2642211914062, "completions/min_length": 694.25, "completions/min_terminated_length": 694.25, "epoch": 0.1406915092226122, "grad_norm": 0.08696173876523972, "kl": 0.0784912109375, "learning_rate": 1.9866433320848793e-05, "loss": 0.0052, "num_tokens": 238584367.0, "reward": 0.6763393133878708, "reward_std": 0.11629410088062286, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.318359836935997, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 471 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 1012.3594055175781, "completions/mean_terminated_length": 917.20703125, "completions/min_length": 696.75, "completions/min_terminated_length": 696.75, "epoch": 0.14099021731013367, "grad_norm": 0.07271873950958252, "kl": 0.0826416015625, "learning_rate": 1.986465053714199e-05, "loss": 0.0036, "num_tokens": 239104304.0, "reward": 0.6467634290456772, "reward_std": 0.12161009572446346, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.356712743639946, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 472 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 979.0, "completions/mean_length": 1012.8728179931641, "completions/mean_terminated_length": 904.1895751953125, "completions/min_length": 670.5, "completions/min_terminated_length": 670.5, "epoch": 0.14128892539765514, "grad_norm": 0.09634844958782196, "kl": 0.0799560546875, "learning_rate": 1.9862856015372315e-05, "loss": 0.0041, "num_tokens": 239638023.0, "reward": 0.6233259290456772, "reward_std": 0.17226552963256836, "rewards/accuracy_reward/mean": 0.1339285708963871, "rewards/accuracy_reward/std": 0.334286168217659, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.0494418740272522, "step": 473 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9575892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 1020.3795013427734, "completions/mean_terminated_length": 939.9187469482422, "completions/min_length": 815.75, "completions/min_terminated_length": 815.75, "epoch": 0.1415876334851766, "grad_norm": 0.10586819797754288, "kl": 0.086669921875, "learning_rate": 1.9861049757675087e-05, "loss": 0.0037, "num_tokens": 240169937.0, "reward": 0.6060268059372902, "reward_std": 0.15652670711278915, "rewards/accuracy_reward/mean": 0.12276785564608872, "rewards/accuracy_reward/std": 0.28949169628322124, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06181654520332813, "step": 474 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 1017.9821929931641, "completions/mean_terminated_length": 948.1795043945312, "completions/min_length": 829.25, "completions/min_terminated_length": 829.25, "epoch": 0.14188634157269808, "grad_norm": 0.08248304575681686, "kl": 0.078857421875, "learning_rate": 1.98592317661996e-05, "loss": 0.0045, "num_tokens": 240693113.0, "reward": 0.8253348469734192, "reward_std": 0.1438891887664795, "rewards/accuracy_reward/mean": 0.3325892761349678, "rewards/accuracy_reward/std": 0.44123028218746185, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 475 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.75, "completions/mean_length": 1013.0379943847656, "completions/mean_terminated_length": 889.3278045654297, "completions/min_length": 743.5, "completions/min_terminated_length": 743.5, "epoch": 0.14218504966021955, "grad_norm": 0.08174704760313034, "kl": 0.078125, "learning_rate": 1.985740204310909e-05, "loss": 0.0035, "num_tokens": 241223322.0, "reward": 0.5658482313156128, "reward_std": 0.12626874074339867, "rewards/accuracy_reward/mean": 0.07366071315482259, "rewards/accuracy_reward/std": 0.24921715259552002, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 476 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 1013.0156707763672, "completions/mean_terminated_length": 903.8140563964844, "completions/min_length": 683.25, "completions/min_terminated_length": 683.25, "epoch": 0.14248375774774102, "grad_norm": 0.0958942398428917, "kl": 0.076171875, "learning_rate": 1.985556059058078e-05, "loss": 0.0019, "num_tokens": 241743697.0, "reward": 0.6986607313156128, "reward_std": 0.17311900481581688, "rewards/accuracy_reward/mean": 0.20535714086145163, "rewards/accuracy_reward/std": 0.3733350709080696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 477 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9107142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 1015.3795013427734, "completions/mean_terminated_length": 943.0970001220703, "completions/min_length": 829.5, "completions/min_terminated_length": 829.5, "epoch": 0.1427824658352625, "grad_norm": 0.08856062591075897, "kl": 0.0791015625, "learning_rate": 1.9853707410805825e-05, "loss": 0.0023, "num_tokens": 242270219.0, "reward": 0.6261160969734192, "reward_std": 0.1538190357387066, "rewards/accuracy_reward/mean": 0.13169643050059676, "rewards/accuracy_reward/std": 0.3162720501422882, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 478 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.75, "completions/mean_length": 1012.0893249511719, "completions/mean_terminated_length": 901.096923828125, "completions/min_length": 734.5, "completions/min_terminated_length": 734.5, "epoch": 0.14308117392278397, "grad_norm": 0.07044459879398346, "kl": 0.07196044921875, "learning_rate": 1.9851842505989355e-05, "loss": 0.0042, "num_tokens": 242794387.0, "reward": 0.643973246216774, "reward_std": 0.13540462404489517, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.32604630291461945, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03161557391285896, "step": 479 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 1008.7879791259766, "completions/mean_terminated_length": 892.7848205566406, "completions/min_length": 752.5, "completions/min_terminated_length": 752.5, "epoch": 0.14337988201030544, "grad_norm": 0.08245354890823364, "kl": 0.0819091796875, "learning_rate": 1.984996587835043e-05, "loss": 0.0034, "num_tokens": 243321956.0, "reward": 0.6054687798023224, "reward_std": 0.1090904981829226, "rewards/accuracy_reward/mean": 0.118675597012043, "rewards/accuracy_reward/std": 0.27353163063526154, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 480 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 1021.263427734375, "completions/mean_terminated_length": 942.9541931152344, "completions/min_length": 824.25, "completions/min_terminated_length": 824.25, "epoch": 0.1436785900978269, "grad_norm": 0.07543475180864334, "kl": 0.0755615234375, "learning_rate": 1.9848077530122083e-05, "loss": 0.0038, "num_tokens": 243860618.0, "reward": 0.6467634290456772, "reward_std": 0.17136552929878235, "rewards/accuracy_reward/mean": 0.15178571455180645, "rewards/accuracy_reward/std": 0.3420974537730217, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.029207061510533094, "step": 481 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 1002.1094207763672, "completions/mean_terminated_length": 881.8417053222656, "completions/min_length": 612.25, "completions/min_terminated_length": 612.25, "epoch": 0.14397729818534838, "grad_norm": 0.09694728255271912, "kl": 0.0792236328125, "learning_rate": 1.9846177463551273e-05, "loss": 0.0038, "num_tokens": 244379355.0, "reward": 0.726004496216774, "reward_std": 0.1953349970281124, "rewards/accuracy_reward/mean": 0.2321428544819355, "rewards/accuracy_reward/std": 0.4181303307414055, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.030388458166271448, "step": 482 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 1014.9062957763672, "completions/mean_terminated_length": 923.1999969482422, "completions/min_length": 729.5, "completions/min_terminated_length": 729.5, "epoch": 0.14427600627286985, "grad_norm": 0.0820012018084526, "kl": 0.0787353515625, "learning_rate": 1.9844265680898917e-05, "loss": 0.0047, "num_tokens": 244906689.0, "reward": 0.6941964626312256, "reward_std": 0.15021428279578686, "rewards/accuracy_reward/mean": 0.20089285727590322, "rewards/accuracy_reward/std": 0.3748129904270172, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 483 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 999.4821929931641, "completions/mean_terminated_length": 841.1521301269531, "completions/min_length": 588.5, "completions/min_terminated_length": 588.5, "epoch": 0.14457471436039132, "grad_norm": 0.0856703594326973, "kl": 0.072265625, "learning_rate": 1.9842342184439865e-05, "loss": 0.003, "num_tokens": 245428281.0, "reward": 0.680245578289032, "reward_std": 0.13217167742550373, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3774593621492386, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04304015310481191, "step": 484 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 1016.5960083007812, "completions/mean_terminated_length": 931.0633392333984, "completions/min_length": 770.0, "completions/min_terminated_length": 770.0, "epoch": 0.14487342244791276, "grad_norm": 0.08536501228809357, "kl": 0.0771484375, "learning_rate": 1.984040697646291e-05, "loss": 0.0049, "num_tokens": 245966308.0, "reward": 0.6796875149011612, "reward_std": 0.20038382709026337, "rewards/accuracy_reward/mean": 0.1852678619325161, "rewards/accuracy_reward/std": 0.37084808573126793, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 485 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.75, "completions/mean_length": 1014.3214569091797, "completions/mean_terminated_length": 904.3090362548828, "completions/min_length": 743.0, "completions/min_terminated_length": 743.0, "epoch": 0.14517213053543424, "grad_norm": 0.07414808869361877, "kl": 0.0758056640625, "learning_rate": 1.9838460059270775e-05, "loss": 0.0017, "num_tokens": 246491316.0, "reward": 0.6222098469734192, "reward_std": 0.10877443104982376, "rewards/accuracy_reward/mean": 0.12500000558793545, "rewards/accuracy_reward/std": 0.32829907536506653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 486 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 1007.575927734375, "completions/mean_terminated_length": 887.0184631347656, "completions/min_length": 742.0, "completions/min_terminated_length": 742.0, "epoch": 0.1454708386229557, "grad_norm": 0.0839051827788353, "kl": 0.080078125, "learning_rate": 1.9836501435180122e-05, "loss": 0.0073, "num_tokens": 247012966.0, "reward": 0.6395089626312256, "reward_std": 0.19010674953460693, "rewards/accuracy_reward/mean": 0.15215774113312364, "rewards/accuracy_reward/std": 0.33260807022452354, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.03659330680966377, "step": 487 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.5, "completions/mean_length": 1000.3616333007812, "completions/mean_terminated_length": 853.5520935058594, "completions/min_length": 572.5, "completions/min_terminated_length": 572.5, "epoch": 0.14576954671047718, "grad_norm": 0.09101524204015732, "kl": 0.0780029296875, "learning_rate": 1.983453110652154e-05, "loss": 0.0022, "num_tokens": 247534856.0, "reward": 0.6785714477300644, "reward_std": 0.16454324685037136, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3809955567121506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 488 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.921875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 1011.7946929931641, "completions/mean_terminated_length": 888.1864013671875, "completions/min_length": 729.75, "completions/min_terminated_length": 729.75, "epoch": 0.14606825479799865, "grad_norm": 0.08172687888145447, "kl": 0.07611083984375, "learning_rate": 1.983254907563955e-05, "loss": 0.0028, "num_tokens": 248054668.0, "reward": 0.7633928954601288, "reward_std": 0.19896897487342358, "rewards/accuracy_reward/mean": 0.2723214253783226, "rewards/accuracy_reward/std": 0.4452969804406166, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03946960438042879, "step": 489 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.25, "completions/mean_length": 996.8504638671875, "completions/mean_terminated_length": 836.7043151855469, "completions/min_length": 558.25, "completions/min_terminated_length": 558.25, "epoch": 0.14636696288552012, "grad_norm": 0.07622019946575165, "kl": 0.0804443359375, "learning_rate": 1.9830555344892585e-05, "loss": 0.0019, "num_tokens": 248576281.0, "reward": 0.5831473618745804, "reward_std": 0.0996546489186585, "rewards/accuracy_reward/mean": 0.09151785750873387, "rewards/accuracy_reward/std": 0.21003371104598045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04334343643859029, "step": 490 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.75, "completions/mean_length": 1019.107177734375, "completions/mean_terminated_length": 918.8583679199219, "completions/min_length": 826.5, "completions/min_terminated_length": 826.5, "epoch": 0.1466656709730416, "grad_norm": 0.09845522791147232, "kl": 0.079833984375, "learning_rate": 1.9828549916653013e-05, "loss": 0.0035, "num_tokens": 249104345.0, "reward": 0.6116071790456772, "reward_std": 0.16328992694616318, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.3221179097890854, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.044330806471407413, "step": 491 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7723214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 990.0201263427734, "completions/mean_terminated_length": 883.5090179443359, "completions/min_length": 650.5, "completions/min_terminated_length": 650.5, "epoch": 0.14696437906056306, "grad_norm": 0.08002743124961853, "kl": 0.075439453125, "learning_rate": 1.982653279330712e-05, "loss": 0.0084, "num_tokens": 249629506.0, "reward": 0.7974330633878708, "reward_std": 0.1306764930486679, "rewards/accuracy_reward/mean": 0.3035714216530323, "rewards/accuracy_reward/std": 0.4352741166949272, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616156578064, "rewards/tag_count_reward/std": 0.036427486687898636, "step": 492 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8504464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 1003.9018249511719, "completions/mean_terminated_length": 901.2093658447266, "completions/min_length": 720.25, "completions/min_terminated_length": 720.25, "epoch": 0.14726308714808453, "grad_norm": 0.07164473831653595, "kl": 0.0787353515625, "learning_rate": 1.98245039772551e-05, "loss": 0.0052, "num_tokens": 250158518.0, "reward": 0.6406250149011612, "reward_std": 0.0859872568398714, "rewards/accuracy_reward/mean": 0.14657737966626883, "rewards/accuracy_reward/std": 0.334584828466177, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 493 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.25, "completions/mean_length": 988.091552734375, "completions/mean_terminated_length": 869.9938201904297, "completions/min_length": 655.0, "completions/min_terminated_length": 655.0, "epoch": 0.147561795235606, "grad_norm": 0.07477010041475296, "kl": 0.07958984375, "learning_rate": 1.9822463470911068e-05, "loss": 0.0072, "num_tokens": 250682655.0, "reward": 0.6618303954601288, "reward_std": 0.13317213580012321, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.32190990820527077, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 494 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 1002.7232818603516, "completions/mean_terminated_length": 859.874267578125, "completions/min_length": 605.5, "completions/min_terminated_length": 605.5, "epoch": 0.14786050332312747, "grad_norm": 0.08721668273210526, "kl": 0.0863037109375, "learning_rate": 1.982041127670304e-05, "loss": 0.0055, "num_tokens": 251206083.0, "reward": 0.6914062798023224, "reward_std": 0.14985758624970913, "rewards/accuracy_reward/mean": 0.1986607164144516, "rewards/accuracy_reward/std": 0.3841792270541191, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04085774393752217, "step": 495 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 1010.825927734375, "completions/mean_terminated_length": 911.2389221191406, "completions/min_length": 744.75, "completions/min_terminated_length": 744.75, "epoch": 0.14815921141064894, "grad_norm": 0.07105280458927155, "kl": 0.0877685546875, "learning_rate": 1.9818347397072954e-05, "loss": 0.0036, "num_tokens": 251732117.0, "reward": 0.604910746216774, "reward_std": 0.11923303129151464, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.26314349472522736, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.031776280142366886, "step": 496 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.90625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 1012.4107513427734, "completions/mean_terminated_length": 924.7005462646484, "completions/min_length": 779.0, "completions/min_terminated_length": 779.0, "epoch": 0.14845791949817042, "grad_norm": 0.07289285957813263, "kl": 0.0811767578125, "learning_rate": 1.9816271834476642e-05, "loss": 0.0033, "num_tokens": 252262381.0, "reward": 0.6930803805589676, "reward_std": 0.16317995823919773, "rewards/accuracy_reward/mean": 0.19642856530845165, "rewards/accuracy_reward/std": 0.39161213487386703, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.023462072014808655, "step": 497 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.75, "completions/mean_length": 1014.8370971679688, "completions/mean_terminated_length": 921.3488311767578, "completions/min_length": 773.25, "completions/min_terminated_length": 773.25, "epoch": 0.1487566275856919, "grad_norm": 0.08507653325796127, "kl": 0.0855712890625, "learning_rate": 1.9814184591383835e-05, "loss": 0.0027, "num_tokens": 252788340.0, "reward": 0.6919643133878708, "reward_std": 0.14264661632478237, "rewards/accuracy_reward/mean": 0.1986607201397419, "rewards/accuracy_reward/std": 0.39499498158693314, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 498 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 1012.4420166015625, "completions/mean_terminated_length": 891.3125152587891, "completions/min_length": 635.25, "completions/min_terminated_length": 635.25, "epoch": 0.14905533567321336, "grad_norm": 0.09686776250600815, "kl": 0.084716796875, "learning_rate": 1.981208567027818e-05, "loss": 0.0072, "num_tokens": 253314666.0, "reward": 0.6796875298023224, "reward_std": 0.18516811914741993, "rewards/accuracy_reward/mean": 0.19233630853705108, "rewards/accuracy_reward/std": 0.322110403329134, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 499 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 1009.5580902099609, "completions/mean_terminated_length": 892.6847686767578, "completions/min_length": 726.25, "completions/min_terminated_length": 726.25, "epoch": 0.14935404376073483, "grad_norm": 0.08366283774375916, "kl": 0.0848388671875, "learning_rate": 1.9809975073657192e-05, "loss": 0.0043, "num_tokens": 253851108.0, "reward": 0.5686384066939354, "reward_std": 0.10330313816666603, "rewards/accuracy_reward/mean": 0.07589285634458065, "rewards/accuracy_reward/std": 0.22398513182997704, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03406794602051377, "step": 500 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 970.25, "completions/mean_length": 1016.9933471679688, "completions/mean_terminated_length": 904.8500213623047, "completions/min_length": 791.0, "completions/min_terminated_length": 791.0, "epoch": 0.1496527518482563, "grad_norm": 0.09645885974168777, "kl": 0.0863037109375, "learning_rate": 1.9807852804032306e-05, "loss": 0.0046, "num_tokens": 254371153.0, "reward": 0.6289062798023224, "reward_std": 0.14345258846879005, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.31716160476207733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05163817573338747, "step": 501 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.25, "completions/mean_length": 1013.1853179931641, "completions/mean_terminated_length": 907.1470794677734, "completions/min_length": 784.75, "completions/min_terminated_length": 784.75, "epoch": 0.14995145993577777, "grad_norm": 0.07315116375684738, "kl": 0.0830078125, "learning_rate": 1.980571886392883e-05, "loss": 0.0033, "num_tokens": 254895716.0, "reward": 0.5753348618745804, "reward_std": 0.10664537106640637, "rewards/accuracy_reward/mean": 0.08258928544819355, "rewards/accuracy_reward/std": 0.23325209319591522, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 502 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9441964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 1017.997802734375, "completions/mean_terminated_length": 918.2166900634766, "completions/min_length": 795.0, "completions/min_terminated_length": 795.0, "epoch": 0.15025016802329924, "grad_norm": 0.09489566087722778, "kl": 0.083984375, "learning_rate": 1.9803573255885967e-05, "loss": 0.0059, "num_tokens": 255424115.0, "reward": 0.6473214626312256, "reward_std": 0.18016865104436874, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.34905095770955086, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 503 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 733.0, "completions/mean_length": 1015.513427734375, "completions/mean_terminated_length": 661.8250122070312, "completions/min_length": 802.75, "completions/min_terminated_length": 546.75, "epoch": 0.1505488761108207, "grad_norm": 0.08013828843832016, "kl": 0.081298828125, "learning_rate": 1.9801415982456803e-05, "loss": 0.003, "num_tokens": 255944905.0, "reward": 0.691964328289032, "reward_std": 0.14372770977206528, "rewards/accuracy_reward/mean": 0.1986607164144516, "rewards/accuracy_reward/std": 0.3950059413909912, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.037730947602540255, "step": 504 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.5, "completions/mean_length": 995.6629943847656, "completions/mean_terminated_length": 805.3278045654297, "completions/min_length": 443.75, "completions/min_terminated_length": 443.75, "epoch": 0.15084758419834218, "grad_norm": 0.08850489556789398, "kl": 0.074951171875, "learning_rate": 1.9799247046208297e-05, "loss": 0.0033, "num_tokens": 256463538.0, "reward": 0.6668527126312256, "reward_std": 0.18328820168972015, "rewards/accuracy_reward/mean": 0.1886160708963871, "rewards/accuracy_reward/std": 0.3762248083949089, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04571862844750285, "step": 505 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.0, "completions/mean_length": 1017.3036041259766, "completions/mean_terminated_length": 888.4375152587891, "completions/min_length": 787.0, "completions/min_terminated_length": 787.0, "epoch": 0.15114629228586363, "grad_norm": 0.08073276281356812, "kl": 0.08349609375, "learning_rate": 1.9797066449721295e-05, "loss": 0.0049, "num_tokens": 256996698.0, "reward": 0.6121652126312256, "reward_std": 0.14302975311875343, "rewards/accuracy_reward/mean": 0.12053571175783873, "rewards/accuracy_reward/std": 0.31135473027825356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 506 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 968.0, "completions/mean_length": 1003.0893249511719, "completions/mean_terminated_length": 863.4601898193359, "completions/min_length": 649.75, "completions/min_terminated_length": 649.75, "epoch": 0.1514450003733851, "grad_norm": 0.08896652609109879, "kl": 0.077880859375, "learning_rate": 1.9794874195590514e-05, "loss": 0.0045, "num_tokens": 257520226.0, "reward": 0.6484375298023224, "reward_std": 0.14213804341852665, "rewards/accuracy_reward/mean": 0.16071428451687098, "rewards/accuracy_reward/std": 0.350029319524765, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 507 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 971.75, "completions/mean_length": 1016.0736999511719, "completions/mean_terminated_length": 884.84228515625, "completions/min_length": 684.5, "completions/min_terminated_length": 684.5, "epoch": 0.15174370846090657, "grad_norm": 0.08418789505958557, "kl": 0.082275390625, "learning_rate": 1.9792670286424546e-05, "loss": 0.0056, "num_tokens": 258047667.0, "reward": 0.617745578289032, "reward_std": 0.1844916883856058, "rewards/accuracy_reward/mean": 0.13578869169577956, "rewards/accuracy_reward/std": 0.30205147713422775, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196588039398, "step": 508 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 1017.9978179931641, "completions/mean_terminated_length": 925.6358489990234, "completions/min_length": 819.5, "completions/min_terminated_length": 819.5, "epoch": 0.15204241654842804, "grad_norm": 0.08661338686943054, "kl": 0.0784912109375, "learning_rate": 1.979045472484584e-05, "loss": 0.0034, "num_tokens": 258573890.0, "reward": 0.6082589626312256, "reward_std": 0.1446664035320282, "rewards/accuracy_reward/mean": 0.11607143050059676, "rewards/accuracy_reward/std": 0.2816254049539566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 509 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 740.75, "completions/mean_length": 1020.2321624755859, "completions/mean_terminated_length": 693.1934661865234, "completions/min_length": 879.5, "completions/min_terminated_length": 623.5, "epoch": 0.1523411246359495, "grad_norm": 0.0764097273349762, "kl": 0.0792236328125, "learning_rate": 1.9788227513490724e-05, "loss": 0.0035, "num_tokens": 259103258.0, "reward": 0.5892857313156128, "reward_std": 0.09837665222585201, "rewards/accuracy_reward/mean": 0.09598214458674192, "rewards/accuracy_reward/std": 0.28579577803611755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 510 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.5, "completions/mean_length": 1017.7120971679688, "completions/mean_terminated_length": 932.3039855957031, "completions/min_length": 764.5, "completions/min_terminated_length": 764.5, "epoch": 0.15263983272347098, "grad_norm": 0.07068465650081635, "kl": 0.0791015625, "learning_rate": 1.9785988655009386e-05, "loss": 0.0027, "num_tokens": 259625177.0, "reward": 0.5987723544239998, "reward_std": 0.11692853644490242, "rewards/accuracy_reward/mean": 0.10416666511446238, "rewards/accuracy_reward/std": 0.24066011980175972, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937574505806, "rewards/tag_count_reward/std": 0.024776804260909557, "step": 511 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 1005.4174652099609, "completions/mean_terminated_length": 925.0559997558594, "completions/min_length": 790.75, "completions/min_terminated_length": 790.75, "epoch": 0.15293854081099245, "grad_norm": 0.07000309973955154, "kl": 0.080322265625, "learning_rate": 1.9783738152065862e-05, "loss": 0.0017, "num_tokens": 260150308.0, "reward": 0.6674107313156128, "reward_std": 0.10764512978494167, "rewards/accuracy_reward/mean": 0.16964285727590322, "rewards/accuracy_reward/std": 0.36074451357126236, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 512 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 1016.5692291259766, "completions/mean_terminated_length": 938.9330444335938, "completions/min_length": 792.75, "completions/min_terminated_length": 792.75, "epoch": 0.15323724889851392, "grad_norm": 0.08282462507486343, "kl": 0.079833984375, "learning_rate": 1.9781476007338058e-05, "loss": 0.0035, "num_tokens": 260672035.0, "reward": 0.5697544813156128, "reward_std": 0.08588057057932019, "rewards/accuracy_reward/mean": 0.07589285634458065, "rewards/accuracy_reward/std": 0.22128358110785484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 513 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 1000.6317596435547, "completions/mean_terminated_length": 891.9998321533203, "completions/min_length": 721.75, "completions/min_terminated_length": 721.75, "epoch": 0.1535359569860354, "grad_norm": 0.06470706313848495, "kl": 0.077880859375, "learning_rate": 1.9779202223517725e-05, "loss": 0.0033, "num_tokens": 261190110.0, "reward": 0.8058036118745804, "reward_std": 0.12474093679338694, "rewards/accuracy_reward/mean": 0.3080357052385807, "rewards/accuracy_reward/std": 0.449780710041523, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 514 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.90625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 1013.2745971679688, "completions/mean_terminated_length": 924.0422973632812, "completions/min_length": 705.5, "completions/min_terminated_length": 705.5, "epoch": 0.15383466507355686, "grad_norm": 0.07606718689203262, "kl": 0.0712890625, "learning_rate": 1.9776916803310463e-05, "loss": 0.0013, "num_tokens": 261717849.0, "reward": 0.6199776977300644, "reward_std": 0.14289005380123854, "rewards/accuracy_reward/mean": 0.12723214109428227, "rewards/accuracy_reward/std": 0.29832659289240837, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 515 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9263392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.75, "completions/mean_length": 1012.4821929931641, "completions/mean_terminated_length": 886.0625152587891, "completions/min_length": 681.25, "completions/min_terminated_length": 681.25, "epoch": 0.15413337316107834, "grad_norm": 0.07026064395904541, "kl": 0.081787109375, "learning_rate": 1.977461974943572e-05, "loss": 0.0033, "num_tokens": 262240577.0, "reward": 0.5753348618745804, "reward_std": 0.08858280442655087, "rewards/accuracy_reward/mean": 0.07812500023283064, "rewards/accuracy_reward/std": 0.24501014314591885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.01845060009509325, "step": 516 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 1013.0134429931641, "completions/mean_terminated_length": 911.4238128662109, "completions/min_length": 776.75, "completions/min_terminated_length": 776.75, "epoch": 0.1544320812485998, "grad_norm": 0.08984305709600449, "kl": 0.0860595703125, "learning_rate": 1.977231106462679e-05, "loss": 0.0042, "num_tokens": 262772903.0, "reward": 0.766183078289032, "reward_std": 0.20067015662789345, "rewards/accuracy_reward/mean": 0.2700892873108387, "rewards/accuracy_reward/std": 0.40934182703495026, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 517 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 735.25, "completions/mean_length": 1020.1585083007812, "completions/mean_terminated_length": 685.3392944335938, "completions/min_length": 863.75, "completions/min_terminated_length": 607.75, "epoch": 0.15473078933612128, "grad_norm": 0.09802397340536118, "kl": 0.0882568359375, "learning_rate": 1.9769990751630805e-05, "loss": 0.0044, "num_tokens": 263310574.0, "reward": 0.6054687798023224, "reward_std": 0.11482463963329792, "rewards/accuracy_reward/mean": 0.11830356996506453, "rewards/accuracy_reward/std": 0.2952413931488991, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054666440933942795, "step": 518 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.5, "completions/mean_length": 1015.8214721679688, "completions/mean_terminated_length": 911.4875183105469, "completions/min_length": 671.0, "completions/min_terminated_length": 671.0, "epoch": 0.15502949742364275, "grad_norm": 0.10368185490369797, "kl": 0.07958984375, "learning_rate": 1.9767658813208725e-05, "loss": 0.0037, "num_tokens": 263834782.0, "reward": 0.681919664144516, "reward_std": 0.18981481716036797, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.4027089551091194, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05723519250750542, "step": 519 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 983.5, "completions/mean_length": 1021.2634582519531, "completions/mean_terminated_length": 957.2361297607422, "completions/min_length": 886.75, "completions/min_terminated_length": 886.75, "epoch": 0.15532820551116422, "grad_norm": 0.10740809142589569, "kl": 0.0865478515625, "learning_rate": 1.976531525213536e-05, "loss": 0.0034, "num_tokens": 264367732.0, "reward": 0.550781287252903, "reward_std": 0.1467338241636753, "rewards/accuracy_reward/mean": 0.07142857322469354, "rewards/accuracy_reward/std": 0.2351885586977005, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526828289032, "rewards/tag_count_reward/std": 0.0668574133887887, "step": 520 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.986607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 642.5, "completions/mean_length": 1022.0960083007812, "completions/mean_terminated_length": 626.75, "completions/min_length": 861.5, "completions/min_terminated_length": 605.5, "epoch": 0.1556269135986857, "grad_norm": 0.09917692095041275, "kl": 0.083984375, "learning_rate": 1.9762960071199334e-05, "loss": 0.0038, "num_tokens": 264906895.0, "reward": 0.6618303805589676, "reward_std": 0.17766303196549416, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.3678515702486038, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267761349678, "rewards/tag_count_reward/std": 0.0654488280415535, "step": 521 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 987.0, "completions/mean_length": 1020.0737152099609, "completions/mean_terminated_length": 896.1250152587891, "completions/min_length": 789.75, "completions/min_terminated_length": 789.75, "epoch": 0.15592562168620716, "grad_norm": 0.10272770375013351, "kl": 0.0836181640625, "learning_rate": 1.9760593273203107e-05, "loss": 0.0039, "num_tokens": 265429392.0, "reward": 0.6205357313156128, "reward_std": 0.15831419453024864, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.3372400626540184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4776785671710968, "rewards/tag_count_reward/std": 0.06732531171292067, "step": 522 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.953125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 1017.6652221679688, "completions/mean_terminated_length": 886.9697113037109, "completions/min_length": 688.25, "completions/min_terminated_length": 688.25, "epoch": 0.15622432977372863, "grad_norm": 0.10394606739282608, "kl": 0.078369140625, "learning_rate": 1.9758214860962965e-05, "loss": 0.004, "num_tokens": 265960954.0, "reward": 0.6685267984867096, "reward_std": 0.19121186528354883, "rewards/accuracy_reward/mean": 0.18303571012802422, "rewards/accuracy_reward/std": 0.337903356179595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05805058777332306, "step": 523 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 985.75, "completions/mean_length": 1019.2902069091797, "completions/mean_terminated_length": 938.8317413330078, "completions/min_length": 871.5, "completions/min_terminated_length": 871.5, "epoch": 0.1565230378612501, "grad_norm": 0.0947079211473465, "kl": 0.0804443359375, "learning_rate": 1.9755824837308996e-05, "loss": 0.0038, "num_tokens": 266484940.0, "reward": 0.5965401977300644, "reward_std": 0.15424208715558052, "rewards/accuracy_reward/mean": 0.1116071455180645, "rewards/accuracy_reward/std": 0.2625268027186394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.058686986565589905, "step": 524 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.984375, "completions/max_length": 1024.0, "completions/max_terminated_length": 639.25, "completions/mean_length": 1021.7857513427734, "completions/mean_terminated_length": 623.6666717529297, "completions/min_length": 863.25, "completions/min_terminated_length": 607.25, "epoch": 0.15682174594877157, "grad_norm": 0.09812161326408386, "kl": 0.0828857421875, "learning_rate": 1.9753423205085127e-05, "loss": 0.004, "num_tokens": 267020268.0, "reward": 0.6422991305589676, "reward_std": 0.14274052530527115, "rewards/accuracy_reward/mean": 0.15625000279396772, "rewards/accuracy_reward/std": 0.310007993131876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.055125197395682335, "step": 525 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 952.5, "completions/mean_length": 1017.6473693847656, "completions/mean_terminated_length": 850.1726379394531, "completions/min_length": 739.0, "completions/min_terminated_length": 739.0, "epoch": 0.15712045403629304, "grad_norm": 0.07156387716531754, "kl": 0.0797119140625, "learning_rate": 1.975100996714909e-05, "loss": 0.0024, "num_tokens": 267545646.0, "reward": 0.6261160969734192, "reward_std": 0.13469263166189194, "rewards/accuracy_reward/mean": 0.1294642873108387, "rewards/accuracy_reward/std": 0.33389057219028473, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 526 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 981.75, "completions/mean_length": 1020.2745819091797, "completions/mean_terminated_length": 929.4791717529297, "completions/min_length": 854.0, "completions/min_terminated_length": 854.0, "epoch": 0.15741916212381452, "grad_norm": 0.07178548723459244, "kl": 0.0806884765625, "learning_rate": 1.974858512637241e-05, "loss": 0.0019, "num_tokens": 268076345.0, "reward": 0.608816996216774, "reward_std": 0.11939221434295177, "rewards/accuracy_reward/mean": 0.11941964412108064, "rewards/accuracy_reward/std": 0.3047192357480526, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 527 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 1014.5491333007812, "completions/mean_terminated_length": 912.4125213623047, "completions/min_length": 744.75, "completions/min_terminated_length": 744.75, "epoch": 0.15771787021133596, "grad_norm": 0.08151645958423615, "kl": 0.076416015625, "learning_rate": 1.974614868564045e-05, "loss": 0.0026, "num_tokens": 268602607.0, "reward": 0.6852678954601288, "reward_std": 0.13674399629235268, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3860573023557663, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 528 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 930.75, "completions/mean_length": 1016.4420013427734, "completions/mean_terminated_length": 881.3916778564453, "completions/min_length": 801.0, "completions/min_terminated_length": 801.0, "epoch": 0.15801657829885743, "grad_norm": 0.07725459337234497, "kl": 0.0848388671875, "learning_rate": 1.9743700647852356e-05, "loss": 0.0026, "num_tokens": 269143125.0, "reward": 0.6339286118745804, "reward_std": 0.12481002137064934, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.33755479007959366, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027185317594558, "step": 529 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 1018.6607513427734, "completions/mean_terminated_length": 942.1294708251953, "completions/min_length": 842.25, "completions/min_terminated_length": 842.25, "epoch": 0.1583152863863789, "grad_norm": 0.08142272382974625, "kl": 0.0811767578125, "learning_rate": 1.9741241015921067e-05, "loss": 0.0058, "num_tokens": 269668317.0, "reward": 0.6612723618745804, "reward_std": 0.1759279016405344, "rewards/accuracy_reward/mean": 0.16517857206054032, "rewards/accuracy_reward/std": 0.3311469443142414, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.02676480822265148, "step": 530 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.90625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 1012.8594207763672, "completions/mean_terminated_length": 918.244873046875, "completions/min_length": 661.75, "completions/min_terminated_length": 661.75, "epoch": 0.15861399447390037, "grad_norm": 0.07211315631866455, "kl": 0.0859375, "learning_rate": 1.9738769792773338e-05, "loss": 0.0036, "num_tokens": 270190174.0, "reward": 0.6685268133878708, "reward_std": 0.11019318038597703, "rewards/accuracy_reward/mean": 0.1718749962747097, "rewards/accuracy_reward/std": 0.31554684042930603, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517761349678, "rewards/tag_count_reward/std": 0.024942624382674694, "step": 531 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 754.5, "completions/mean_length": 1022.2343902587891, "completions/mean_terminated_length": 715.875, "completions/min_length": 928.0, "completions/min_terminated_length": 672.0, "epoch": 0.15891270256142184, "grad_norm": 0.08756973594427109, "kl": 0.0858154296875, "learning_rate": 1.97362869813497e-05, "loss": 0.0033, "num_tokens": 270718407.0, "reward": 0.5831473618745804, "reward_std": 0.12592787854373455, "rewards/accuracy_reward/mean": 0.09151785611175, "rewards/accuracy_reward/std": 0.23276208527386189, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.042492654640227556, "step": 532 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 1014.3928985595703, "completions/mean_terminated_length": 876.3877716064453, "completions/min_length": 681.75, "completions/min_terminated_length": 681.75, "epoch": 0.1592114106489433, "grad_norm": 0.09073587507009506, "kl": 0.0880126953125, "learning_rate": 1.9733792584604487e-05, "loss": 0.0049, "num_tokens": 271246999.0, "reward": 0.6529018133878708, "reward_std": 0.14686225354671478, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.32727114111185074, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 533 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.953125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 1019.4688110351562, "completions/mean_terminated_length": 937.6625061035156, "completions/min_length": 791.75, "completions/min_terminated_length": 791.75, "epoch": 0.15951011873646478, "grad_norm": 0.08706633746623993, "kl": 0.0889892578125, "learning_rate": 1.9731286605505802e-05, "loss": 0.0027, "num_tokens": 271773017.0, "reward": 0.6149553805589676, "reward_std": 0.1092001348733902, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.3337417468428612, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 534 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.75, "completions/mean_length": 1018.4911041259766, "completions/mean_terminated_length": 938.3353271484375, "completions/min_length": 837.5, "completions/min_terminated_length": 837.5, "epoch": 0.15980882682398626, "grad_norm": 0.08546914905309677, "kl": 0.08447265625, "learning_rate": 1.972876904703554e-05, "loss": 0.0045, "num_tokens": 272296309.0, "reward": 0.5691964477300644, "reward_std": 0.13871702458709478, "rewards/accuracy_reward/mean": 0.08035714272409678, "rewards/accuracy_reward/std": 0.22708947211503983, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.048379197251051664, "step": 535 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9017857142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 1015.2031555175781, "completions/mean_terminated_length": 932.1844329833984, "completions/min_length": 771.5, "completions/min_terminated_length": 771.5, "epoch": 0.16010753491150773, "grad_norm": 0.07452038675546646, "kl": 0.084228515625, "learning_rate": 1.9726239912189382e-05, "loss": 0.0044, "num_tokens": 272822032.0, "reward": 0.6082589477300644, "reward_std": 0.0965728648006916, "rewards/accuracy_reward/mean": 0.11569940391927958, "rewards/accuracy_reward/std": 0.31270214542746544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 536 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 1013.0714721679688, "completions/mean_terminated_length": 889.98291015625, "completions/min_length": 600.5, "completions/min_terminated_length": 600.5, "epoch": 0.1604062429990292, "grad_norm": 0.0777939185500145, "kl": 0.0919189453125, "learning_rate": 1.9723699203976768e-05, "loss": -0.0049, "num_tokens": 273351392.0, "reward": 0.5747768133878708, "reward_std": 0.11074722185730934, "rewards/accuracy_reward/mean": 0.08035714086145163, "rewards/accuracy_reward/std": 0.2275986149907112, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 537 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 1014.6920166015625, "completions/mean_terminated_length": 910.53125, "completions/min_length": 766.5, "completions/min_terminated_length": 766.5, "epoch": 0.16070495108655067, "grad_norm": 0.09124058485031128, "kl": 0.086669921875, "learning_rate": 1.9721146925420916e-05, "loss": 0.0067, "num_tokens": 273874374.0, "reward": 0.7343750298023224, "reward_std": 0.24561601504683495, "rewards/accuracy_reward/mean": 0.23883928544819355, "rewards/accuracy_reward/std": 0.4167693331837654, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357238650322, "rewards/tag_count_reward/std": 0.030682499054819345, "step": 538 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 1014.1116485595703, "completions/mean_terminated_length": 932.4875030517578, "completions/min_length": 820.5, "completions/min_terminated_length": 820.5, "epoch": 0.16100365917407214, "grad_norm": 0.08071383088827133, "kl": 0.0882568359375, "learning_rate": 1.9718583079558814e-05, "loss": 0.0039, "num_tokens": 274391144.0, "reward": 0.7154018133878708, "reward_std": 0.1663680002093315, "rewards/accuracy_reward/mean": 0.2209821417927742, "rewards/accuracy_reward/std": 0.4013756811618805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.03511275444179773, "step": 539 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 1016.7098693847656, "completions/mean_terminated_length": 929.5726013183594, "completions/min_length": 830.5, "completions/min_terminated_length": 830.5, "epoch": 0.1613023672615936, "grad_norm": 0.07191117107868195, "kl": 0.0838623046875, "learning_rate": 1.971600766944121e-05, "loss": 0.0028, "num_tokens": 274913750.0, "reward": 0.737723246216774, "reward_std": 0.13315464928746223, "rewards/accuracy_reward/mean": 0.2433035634458065, "rewards/accuracy_reward/std": 0.42781852930784225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 540 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.25, "completions/mean_length": 999.5915679931641, "completions/mean_terminated_length": 861.9702911376953, "completions/min_length": 646.25, "completions/min_terminated_length": 646.25, "epoch": 0.16160107534911508, "grad_norm": 0.08428211510181427, "kl": 0.080078125, "learning_rate": 1.9713420698132614e-05, "loss": 0.0055, "num_tokens": 275432303.0, "reward": 0.6629464626312256, "reward_std": 0.11645353212952614, "rewards/accuracy_reward/mean": 0.1718749962747097, "rewards/accuracy_reward/std": 0.3720156028866768, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04386132536455989, "step": 541 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.5, "completions/mean_length": 1018.9419860839844, "completions/mean_terminated_length": 957.3500061035156, "completions/min_length": 890.25, "completions/min_terminated_length": 890.25, "epoch": 0.16189978343663655, "grad_norm": 0.07953327149152756, "kl": 0.0892333984375, "learning_rate": 1.9710822168711302e-05, "loss": 0.0036, "num_tokens": 275955509.0, "reward": 0.546316996216774, "reward_std": 0.08152063516899943, "rewards/accuracy_reward/mean": 0.05580357275903225, "rewards/accuracy_reward/std": 0.15478064119815826, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.045375614892691374, "step": 542 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 733.5, "completions/mean_length": 1022.6004638671875, "completions/mean_terminated_length": 707.5500030517578, "completions/min_length": 933.0, "completions/min_terminated_length": 677.0, "epoch": 0.16219849152415802, "grad_norm": 0.08496944606304169, "kl": 0.08837890625, "learning_rate": 1.9708212084269283e-05, "loss": 0.0037, "num_tokens": 276488978.0, "reward": 0.6149553954601288, "reward_std": 0.1464623361825943, "rewards/accuracy_reward/mean": 0.12276786006987095, "rewards/accuracy_reward/std": 0.3189665228128433, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 543 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 977.0, "completions/mean_length": 1003.3080749511719, "completions/mean_terminated_length": 825.0417022705078, "completions/min_length": 624.75, "completions/min_terminated_length": 624.75, "epoch": 0.1624971996116795, "grad_norm": 0.08950407058000565, "kl": 0.08056640625, "learning_rate": 1.9705590447912333e-05, "loss": 0.0022, "num_tokens": 277008972.0, "reward": 0.7165178805589676, "reward_std": 0.17273670434951782, "rewards/accuracy_reward/mean": 0.2351190447807312, "rewards/accuracy_reward/std": 0.42052343487739563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05434367246925831, "step": 544 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 982.5, "completions/mean_length": 1014.0335235595703, "completions/mean_terminated_length": 847.4547271728516, "completions/min_length": 698.75, "completions/min_terminated_length": 698.75, "epoch": 0.16279590769920096, "grad_norm": 0.08934961259365082, "kl": 0.0867919921875, "learning_rate": 1.9702957262759964e-05, "loss": 0.0027, "num_tokens": 277534731.0, "reward": 0.6222098469734192, "reward_std": 0.09669488715007901, "rewards/accuracy_reward/mean": 0.1361607164144516, "rewards/accuracy_reward/std": 0.32614754885435104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.056536297313869, "step": 545 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 973.5, "completions/mean_length": 1013.1205749511719, "completions/mean_terminated_length": 890.4414825439453, "completions/min_length": 762.25, "completions/min_terminated_length": 762.25, "epoch": 0.16309461578672244, "grad_norm": 0.08311325311660767, "kl": 0.0826416015625, "learning_rate": 1.9700312531945444e-05, "loss": 0.003, "num_tokens": 278062289.0, "reward": 0.6914062649011612, "reward_std": 0.13526382856070995, "rewards/accuracy_reward/mean": 0.198660708963871, "rewards/accuracy_reward/std": 0.394012413918972, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03606722131371498, "step": 546 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 974.25, "completions/mean_length": 1021.9196929931641, "completions/mean_terminated_length": 946.9166717529297, "completions/min_length": 906.25, "completions/min_terminated_length": 906.25, "epoch": 0.1633933238742439, "grad_norm": 0.09318860620260239, "kl": 0.085693359375, "learning_rate": 1.9697656258615753e-05, "loss": 0.0029, "num_tokens": 278591053.0, "reward": 0.6400669813156128, "reward_std": 0.2068628892302513, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.3436601236462593, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 547 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9821428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 865.5, "completions/mean_length": 1021.3192291259766, "completions/mean_terminated_length": 820.25, "completions/min_length": 777.25, "completions/min_terminated_length": 777.25, "epoch": 0.16369203196176538, "grad_norm": 0.09132366627454758, "kl": 0.0853271484375, "learning_rate": 1.9694988445931636e-05, "loss": 0.0039, "num_tokens": 279123068.0, "reward": 0.5959821566939354, "reward_std": 0.12592647224664688, "rewards/accuracy_reward/mean": 0.10937499813735485, "rewards/accuracy_reward/std": 0.2632593512535095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05596293695271015, "step": 548 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 1010.3348541259766, "completions/mean_terminated_length": 891.7889099121094, "completions/min_length": 699.5, "completions/min_terminated_length": 699.5, "epoch": 0.16399074004928682, "grad_norm": 0.0847434401512146, "kl": 0.082275390625, "learning_rate": 1.9692309097067546e-05, "loss": 0.0049, "num_tokens": 279643378.0, "reward": 0.7293527126312256, "reward_std": 0.16900473088026047, "rewards/accuracy_reward/mean": 0.2366071455180645, "rewards/accuracy_reward/std": 0.41173024475574493, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 549 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.75, "completions/mean_length": 1018.4129943847656, "completions/mean_terminated_length": 935.1180572509766, "completions/min_length": 845.0, "completions/min_terminated_length": 845.0, "epoch": 0.1642894481368083, "grad_norm": 0.08690521866083145, "kl": 0.082275390625, "learning_rate": 1.9689618215211673e-05, "loss": 0.0033, "num_tokens": 280170603.0, "reward": 0.5814732313156128, "reward_std": 0.13672788254916668, "rewards/accuracy_reward/mean": 0.09449404897168279, "rewards/accuracy_reward/std": 0.27362992241978645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04666052386164665, "step": 550 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.96875, "completions/max_length": 1024.0, "completions/max_terminated_length": 975.25, "completions/mean_length": 1021.029052734375, "completions/mean_terminated_length": 923.8416748046875, "completions/min_length": 851.25, "completions/min_terminated_length": 851.25, "epoch": 0.16458815622432976, "grad_norm": 0.09608875960111618, "kl": 0.085693359375, "learning_rate": 1.9686915803565934e-05, "loss": 0.0039, "num_tokens": 280706344.0, "reward": 0.5608259290456772, "reward_std": 0.11127222701907158, "rewards/accuracy_reward/mean": 0.07812499953433871, "rewards/accuracy_reward/std": 0.24843744561076164, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06313127465546131, "step": 551 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 1017.1339569091797, "completions/mean_terminated_length": 917.1848602294922, "completions/min_length": 759.75, "completions/min_terminated_length": 759.75, "epoch": 0.16488686431185123, "grad_norm": 0.08359185606241226, "kl": 0.0810546875, "learning_rate": 1.9684201865345952e-05, "loss": 0.0046, "num_tokens": 281230564.0, "reward": 0.627232164144516, "reward_std": 0.1362733691930771, "rewards/accuracy_reward/mean": 0.13616071362048388, "rewards/accuracy_reward/std": 0.3239203318953514, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 552 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.75, "completions/mean_length": 1008.6317596435547, "completions/mean_terminated_length": 887.7451934814453, "completions/min_length": 685.0, "completions/min_terminated_length": 685.0, "epoch": 0.1651855723993727, "grad_norm": 0.09307941049337387, "kl": 0.085693359375, "learning_rate": 1.968147640378108e-05, "loss": 0.0017, "num_tokens": 281754671.0, "reward": 0.6071428805589676, "reward_std": 0.13291524350643158, "rewards/accuracy_reward/mean": 0.12351190205663443, "rewards/accuracy_reward/std": 0.3146289065480232, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04420433798804879, "step": 553 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9174107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 1013.7433471679688, "completions/mean_terminated_length": 913.0500335693359, "completions/min_length": 717.5, "completions/min_terminated_length": 717.5, "epoch": 0.16548428048689418, "grad_norm": 0.089441679418087, "kl": 0.083251953125, "learning_rate": 1.9678739422114375e-05, "loss": 0.0052, "num_tokens": 282293324.0, "reward": 0.7555803954601288, "reward_std": 0.20909369736909866, "rewards/accuracy_reward/mean": 0.2611607126891613, "rewards/accuracy_reward/std": 0.41580767184495926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 554 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.5, "completions/mean_length": 1016.9375305175781, "completions/mean_terminated_length": 924.3972320556641, "completions/min_length": 804.0, "completions/min_terminated_length": 804.0, "epoch": 0.16578298857441565, "grad_norm": 0.09433417022228241, "kl": 0.0889892578125, "learning_rate": 1.96759909236026e-05, "loss": 0.0043, "num_tokens": 282832832.0, "reward": 0.6233259290456772, "reward_std": 0.19700909778475761, "rewards/accuracy_reward/mean": 0.14471725933253765, "rewards/accuracy_reward/std": 0.34520065784454346, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05464820470660925, "step": 555 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.5, "completions/mean_length": 1017.8348541259766, "completions/mean_terminated_length": 899.920654296875, "completions/min_length": 752.75, "completions/min_terminated_length": 752.75, "epoch": 0.16608169666193712, "grad_norm": 0.09488268196582794, "kl": 0.0885009765625, "learning_rate": 1.967323091151623e-05, "loss": 0.0039, "num_tokens": 283360662.0, "reward": 0.5954241305589676, "reward_std": 0.17226852104067802, "rewards/accuracy_reward/mean": 0.10863095079548657, "rewards/accuracy_reward/std": 0.2832146417349577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 556 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.5, "completions/mean_length": 1018.3125457763672, "completions/mean_terminated_length": 932.6089477539062, "completions/min_length": 821.5, "completions/min_terminated_length": 821.5, "epoch": 0.1663804047494586, "grad_norm": 0.08703753352165222, "kl": 0.089111328125, "learning_rate": 1.9670459389139433e-05, "loss": 0.0062, "num_tokens": 283888786.0, "reward": 0.5876116380095482, "reward_std": 0.12662985734641552, "rewards/accuracy_reward/mean": 0.1004464291036129, "rewards/accuracy_reward/std": 0.253735676407814, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 557 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.75, "completions/mean_length": 1013.4219055175781, "completions/mean_terminated_length": 912.09375, "completions/min_length": 753.0, "completions/min_terminated_length": 753.0, "epoch": 0.16667911283698006, "grad_norm": 0.08921290934085846, "kl": 0.08740234375, "learning_rate": 1.9667676359770078e-05, "loss": 0.001, "num_tokens": 284411551.0, "reward": 0.630022332072258, "reward_std": 0.16311750374734402, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.3507334589958191, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05545729910954833, "step": 558 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.75, "completions/mean_length": 1020.9442443847656, "completions/mean_terminated_length": 883.6666717529297, "completions/min_length": 797.0, "completions/min_terminated_length": 797.0, "epoch": 0.16697782092450153, "grad_norm": 0.1006939709186554, "kl": 0.094482421875, "learning_rate": 1.966488182671972e-05, "loss": 0.0034, "num_tokens": 284948502.0, "reward": 0.5641741305589676, "reward_std": 0.15934565663337708, "rewards/accuracy_reward/mean": 0.08258928544819355, "rewards/accuracy_reward/std": 0.2246137410402298, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06328126136213541, "step": 559 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 750.75, "completions/mean_length": 1018.5178833007812, "completions/mean_terminated_length": 671.6250152587891, "completions/min_length": 811.5, "completions/min_terminated_length": 555.5, "epoch": 0.167276529012023, "grad_norm": 0.10114209353923798, "kl": 0.0933837890625, "learning_rate": 1.9662075793313614e-05, "loss": 0.0026, "num_tokens": 285478686.0, "reward": 0.6623884290456772, "reward_std": 0.20573647692799568, "rewards/accuracy_reward/mean": 0.18080356903374195, "rewards/accuracy_reward/std": 0.38036108762025833, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06444018986076117, "step": 560 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 964.5, "completions/mean_length": 1017.9777374267578, "completions/mean_terminated_length": 883.2067565917969, "completions/min_length": 714.5, "completions/min_terminated_length": 714.5, "epoch": 0.16757523709954447, "grad_norm": 0.09395328164100647, "kl": 0.08984375, "learning_rate": 1.9659258262890683e-05, "loss": 0.0049, "num_tokens": 286003092.0, "reward": 0.6841518133878708, "reward_std": 0.18547965213656425, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.3547571450471878, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 561 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.75, "completions/mean_length": 1015.4732513427734, "completions/mean_terminated_length": 914.6598968505859, "completions/min_length": 779.25, "completions/min_terminated_length": 779.25, "epoch": 0.16787394518706594, "grad_norm": 0.09787043184041977, "kl": 0.0950927734375, "learning_rate": 1.9656429238803548e-05, "loss": 0.0048, "num_tokens": 286531384.0, "reward": 0.6724330633878708, "reward_std": 0.19447015598416328, "rewards/accuracy_reward/mean": 0.18973214365541935, "rewards/accuracy_reward/std": 0.37701577693223953, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.062408010475337505, "step": 562 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 1010.5424499511719, "completions/mean_terminated_length": 900.4555358886719, "completions/min_length": 729.75, "completions/min_terminated_length": 729.75, "epoch": 0.1681726532745874, "grad_norm": 0.09382779896259308, "kl": 0.0953369140625, "learning_rate": 1.9653588724418492e-05, "loss": 0.0067, "num_tokens": 287053179.0, "reward": 0.6383928954601288, "reward_std": 0.16060900688171387, "rewards/accuracy_reward/mean": 0.15178571455180645, "rewards/accuracy_reward/std": 0.3483637645840645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05514051578938961, "step": 563 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 1018.8348693847656, "completions/mean_terminated_length": 951.6113739013672, "completions/min_length": 863.75, "completions/min_terminated_length": 863.75, "epoch": 0.16847136136210888, "grad_norm": 0.08543120324611664, "kl": 0.094482421875, "learning_rate": 1.9650736723115476e-05, "loss": 0.0055, "num_tokens": 287578657.0, "reward": 0.6010044813156128, "reward_std": 0.14919657073915005, "rewards/accuracy_reward/mean": 0.11160714249126613, "rewards/accuracy_reward/std": 0.2667530830949545, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 564 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 1009.8527069091797, "completions/mean_terminated_length": 924.1325988769531, "completions/min_length": 718.75, "completions/min_terminated_length": 718.75, "epoch": 0.16877006944963036, "grad_norm": 0.09904766082763672, "kl": 0.1053466796875, "learning_rate": 1.964787323828813e-05, "loss": 0.0071, "num_tokens": 288093983.0, "reward": 0.6222098469734192, "reward_std": 0.15141882747411728, "rewards/accuracy_reward/mean": 0.142113097012043, "rewards/accuracy_reward/std": 0.3422050029039383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05178379639983177, "step": 565 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 999.7835388183594, "completions/mean_terminated_length": 873.0267944335938, "completions/min_length": 632.75, "completions/min_terminated_length": 632.75, "epoch": 0.16906877753715183, "grad_norm": 0.09202238917350769, "kl": 0.110595703125, "learning_rate": 1.9644998273343753e-05, "loss": 0.0055, "num_tokens": 288620366.0, "reward": 0.6043526977300644, "reward_std": 0.1632680483162403, "rewards/accuracy_reward/mean": 0.1160714291036129, "rewards/accuracy_reward/std": 0.30196942389011383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 566 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 1006.6920166015625, "completions/mean_terminated_length": 855.7920684814453, "completions/min_length": 492.5, "completions/min_terminated_length": 492.5, "epoch": 0.1693674856246733, "grad_norm": 0.08741366863250732, "kl": 0.1041259765625, "learning_rate": 1.9642111831703294e-05, "loss": -0.0013, "num_tokens": 289137172.0, "reward": 0.6551339477300644, "reward_std": 0.19748862460255623, "rewards/accuracy_reward/mean": 0.16517857182770967, "rewards/accuracy_reward/std": 0.3513106107711792, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.05053963605314493, "step": 567 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.859375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 1002.997802734375, "completions/mean_terminated_length": 869.8865203857422, "completions/min_length": 567.75, "completions/min_terminated_length": 567.75, "epoch": 0.16966619371219477, "grad_norm": 0.1302172988653183, "kl": 0.1048583984375, "learning_rate": 1.9639213916801366e-05, "loss": 0.006, "num_tokens": 289660339.0, "reward": 0.603794664144516, "reward_std": 0.1490960456430912, "rewards/accuracy_reward/mean": 0.11383928591385484, "rewards/accuracy_reward/std": 0.2697841599583626, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048649012576788664, "step": 568 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.78125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 988.5625305175781, "completions/mean_terminated_length": 869.2077941894531, "completions/min_length": 526.5, "completions/min_terminated_length": 526.5, "epoch": 0.16996490179971624, "grad_norm": 0.08071491867303848, "kl": 0.1197509765625, "learning_rate": 1.963630453208623e-05, "loss": 0.0076, "num_tokens": 290179487.0, "reward": 0.70479916036129, "reward_std": 0.1556643508374691, "rewards/accuracy_reward/mean": 0.2120535708963871, "rewards/accuracy_reward/std": 0.4063788130879402, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 569 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6808035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 971.8147735595703, "completions/mean_terminated_length": 864.3367614746094, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "epoch": 0.1702636098872377, "grad_norm": 0.10540489107370377, "kl": 0.113037109375, "learning_rate": 1.96333836810198e-05, "loss": 0.0084, "num_tokens": 290684380.0, "reward": 0.6305803805589676, "reward_std": 0.17707906663417816, "rewards/accuracy_reward/mean": 0.14062499767169356, "rewards/accuracy_reward/std": 0.3013346903026104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.047475868836045265, "step": 570 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7276785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 974.6495971679688, "completions/mean_terminated_length": 847.4818420410156, "completions/min_length": 431.25, "completions/min_terminated_length": 431.25, "epoch": 0.17056231797475915, "grad_norm": 0.08483725041151047, "kl": 0.1246337890625, "learning_rate": 1.963045136707763e-05, "loss": -0.0065, "num_tokens": 291190655.0, "reward": 0.6640625298023224, "reward_std": 0.14085274934768677, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.36985135823488235, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 571 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6941964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 973.5201263427734, "completions/mean_terminated_length": 856.4056091308594, "completions/min_length": 516.75, "completions/min_terminated_length": 516.75, "epoch": 0.17086102606228062, "grad_norm": 0.09994703531265259, "kl": 0.13818359375, "learning_rate": 1.962750759374891e-05, "loss": -0.0005, "num_tokens": 291700632.0, "reward": 0.6975446790456772, "reward_std": 0.16554807499051094, "rewards/accuracy_reward/mean": 0.2098214253783226, "rewards/accuracy_reward/std": 0.3985388055443764, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.055062913335859776, "step": 572 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7165178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 972.6808471679688, "completions/mean_terminated_length": 838.5470581054688, "completions/min_length": 425.5, "completions/min_terminated_length": 425.5, "epoch": 0.1711597341498021, "grad_norm": 0.10984764248132706, "kl": 0.168212890625, "learning_rate": 1.9624552364536472e-05, "loss": 0.0025, "num_tokens": 292207049.0, "reward": 0.6311384290456772, "reward_std": 0.20383594557642937, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.34419769048690796, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057726223953068256, "step": 573 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7834821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 986.9487152099609, "completions/mean_terminated_length": 858.8006134033203, "completions/min_length": 511.0, "completions/min_terminated_length": 511.0, "epoch": 0.17145844223732357, "grad_norm": 0.13601739704608917, "kl": 0.201904296875, "learning_rate": 1.9621585682956785e-05, "loss": -0.0022, "num_tokens": 292720786.0, "reward": 0.5608258992433548, "reward_std": 0.1551988087594509, "rewards/accuracy_reward/mean": 0.0714285708963871, "rewards/accuracy_reward/std": 0.23700017854571342, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.0494418740272522, "step": 574 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7700892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 987.169677734375, "completions/mean_terminated_length": 865.2437591552734, "completions/min_length": 478.0, "completions/min_terminated_length": 478.0, "epoch": 0.17175715032484504, "grad_norm": 0.12230084836483002, "kl": 0.16162109375, "learning_rate": 1.961860755253993e-05, "loss": -0.0065, "num_tokens": 293245518.0, "reward": 0.5943080633878708, "reward_std": 0.14538741577416658, "rewards/accuracy_reward/mean": 0.10937500279396772, "rewards/accuracy_reward/std": 0.30613621324300766, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.06410206481814384, "step": 575 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7455357142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 976.9464721679688, "completions/mean_terminated_length": 833.6988677978516, "completions/min_length": 323.25, "completions/min_terminated_length": 323.25, "epoch": 0.1720558584123665, "grad_norm": 0.11167631298303604, "kl": 0.224365234375, "learning_rate": 1.9615617976829622e-05, "loss": -0.0155, "num_tokens": 293755030.0, "reward": 0.5630580633878708, "reward_std": 0.12341329827904701, "rewards/accuracy_reward/mean": 0.07812499906867743, "rewards/accuracy_reward/std": 0.20448868721723557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05827039107680321, "step": 576 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7633928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 970.7835388183594, "completions/mean_terminated_length": 798.5880432128906, "completions/min_length": 377.75, "completions/min_terminated_length": 377.75, "epoch": 0.17235456649988798, "grad_norm": 0.152152419090271, "kl": 0.38671875, "learning_rate": 1.961261695938319e-05, "loss": -0.0138, "num_tokens": 294259269.0, "reward": 0.6668527126312256, "reward_std": 0.26421402767300606, "rewards/accuracy_reward/mean": 0.19419643096625805, "rewards/accuracy_reward/std": 0.37329326570034027, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4726562425494194, "rewards/tag_count_reward/std": 0.07770059444010258, "step": 577 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 991.5580749511719, "completions/mean_terminated_length": 870.4177398681641, "completions/min_length": 502.5, "completions/min_terminated_length": 502.5, "epoch": 0.17265327458740945, "grad_norm": 0.1642293930053711, "kl": 0.32763671875, "learning_rate": 1.9609604503771585e-05, "loss": -0.012, "num_tokens": 294777775.0, "reward": 0.6489955633878708, "reward_std": 0.18877257779240608, "rewards/accuracy_reward/mean": 0.1785714328289032, "rewards/accuracy_reward/std": 0.3767102584242821, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4704241082072258, "rewards/tag_count_reward/std": 0.08028176240622997, "step": 578 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 953.9732666015625, "completions/mean_terminated_length": 788.4472198486328, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "epoch": 0.17295198267493092, "grad_norm": 0.16910667717456818, "kl": 0.50732421875, "learning_rate": 1.9606580613579352e-05, "loss": -0.037, "num_tokens": 295281795.0, "reward": 0.5781250298023224, "reward_std": 0.21714706346392632, "rewards/accuracy_reward/mean": 0.11830356996506453, "rewards/accuracy_reward/std": 0.3052988275885582, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4598214253783226, "rewards/tag_count_reward/std": 0.09355180524289608, "step": 579 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 916.4129943847656, "completions/mean_terminated_length": 759.1682739257812, "completions/min_length": 232.5, "completions/min_terminated_length": 232.5, "epoch": 0.1732506907624524, "grad_norm": 0.18439659476280212, "kl": 0.6455078125, "learning_rate": 1.960354529240467e-05, "loss": -0.0716, "num_tokens": 295763084.0, "reward": 0.5396205633878708, "reward_std": 0.2059567403048277, "rewards/accuracy_reward/mean": 0.09374999743886292, "rewards/accuracy_reward/std": 0.2673450317233801, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4458705335855484, "rewards/tag_count_reward/std": 0.1042907815426588, "step": 580 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 785.3504791259766, "completions/mean_terminated_length": 651.1170501708984, "completions/min_length": 56.0, "completions/min_terminated_length": 56.0, "epoch": 0.17354939884997386, "grad_norm": 0.9150468707084656, "kl": 1.2802734375, "learning_rate": 1.960049854385929e-05, "loss": -0.1609, "num_tokens": 296193833.0, "reward": 0.4944196715950966, "reward_std": 0.2180157396942377, "rewards/accuracy_reward/mean": 0.07366071408614516, "rewards/accuracy_reward/std": 0.21108953654766083, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4207589253783226, "rewards/tag_count_reward/std": 0.11674016527831554, "step": 581 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 990.8906707763672, "completions/mean_terminated_length": 787.0877075195312, "completions/min_length": 397.75, "completions/min_terminated_length": 397.75, "epoch": 0.17384810693749533, "grad_norm": 0.18545874953269958, "kl": 0.185302734375, "learning_rate": 1.9597440371568576e-05, "loss": -0.0256, "num_tokens": 296719608.0, "reward": 0.5245535969734192, "reward_std": 0.13212261907756329, "rewards/accuracy_reward/mean": 0.05357142840512097, "rewards/accuracy_reward/std": 0.17566965334117413, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4709821417927742, "rewards/tag_count_reward/std": 0.08113007992506027, "step": 582 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9241071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 1010.8705902099609, "completions/mean_terminated_length": 850.9161834716797, "completions/min_length": 487.75, "completions/min_terminated_length": 487.75, "epoch": 0.1741468150250168, "grad_norm": 0.1155271977186203, "kl": 0.1011962890625, "learning_rate": 1.9594370779171484e-05, "loss": -0.0108, "num_tokens": 297239374.0, "reward": 0.5613839626312256, "reward_std": 0.1420314498245716, "rewards/accuracy_reward/mean": 0.09151785518042743, "rewards/accuracy_reward/std": 0.18228224478662014, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4698660746216774, "rewards/tag_count_reward/std": 0.08083963207900524, "step": 583 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8549107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 1003.3259124755859, "completions/mean_terminated_length": 895.9799194335938, "completions/min_length": 621.75, "completions/min_terminated_length": 621.75, "epoch": 0.17444552311253828, "grad_norm": 0.09595538675785065, "kl": 0.1009521484375, "learning_rate": 1.959128977032056e-05, "loss": -0.0013, "num_tokens": 297764288.0, "reward": 0.5736607387661934, "reward_std": 0.13874814845621586, "rewards/accuracy_reward/mean": 0.09821428451687098, "rewards/accuracy_reward/std": 0.2566083073616028, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4754464253783226, "rewards/tag_count_reward/std": 0.07249996159225702, "step": 584 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 1007.3973693847656, "completions/mean_terminated_length": 894.8916931152344, "completions/min_length": 685.5, "completions/min_terminated_length": 685.5, "epoch": 0.17474423120005975, "grad_norm": 0.10458748042583466, "kl": 0.1029052734375, "learning_rate": 1.958819734868193e-05, "loss": 0.0022, "num_tokens": 298284034.0, "reward": 0.5876116454601288, "reward_std": 0.18653600849211216, "rewards/accuracy_reward/mean": 0.10937500302679837, "rewards/accuracy_reward/std": 0.2812089752405882, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366082072258, "rewards/tag_count_reward/std": 0.06975161843001842, "step": 585 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7812500000000001, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 987.8482666015625, "completions/mean_terminated_length": 886.0181732177734, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "epoch": 0.17504293928758122, "grad_norm": 0.09226831048727036, "kl": 0.1064453125, "learning_rate": 1.9585093517935308e-05, "loss": 0.0019, "num_tokens": 298806686.0, "reward": 0.584263414144516, "reward_std": 0.1383665818721056, "rewards/accuracy_reward/mean": 0.10044642561115324, "rewards/accuracy_reward/std": 0.2508474048227072, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169738650322, "rewards/tag_count_reward/std": 0.06147108040750027, "step": 586 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6986607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 980.9710083007812, "completions/mean_terminated_length": 902.1621398925781, "completions/min_length": 625.5, "completions/min_terminated_length": 625.5, "epoch": 0.1753416473751027, "grad_norm": 0.07203701138496399, "kl": 0.10400390625, "learning_rate": 1.9581978281773966e-05, "loss": 0.0024, "num_tokens": 299325361.0, "reward": 0.7226562798023224, "reward_std": 0.11704868637025356, "rewards/accuracy_reward/mean": 0.22544642770662904, "rewards/accuracy_reward/std": 0.3698040507733822, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 587 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6540178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 962.3303985595703, "completions/mean_terminated_length": 863.0092468261719, "completions/min_length": 585.5, "completions/min_terminated_length": 585.5, "epoch": 0.17564035546262416, "grad_norm": 0.07590491324663162, "kl": 0.1038818359375, "learning_rate": 1.9578851643904776e-05, "loss": 0.0044, "num_tokens": 299830645.0, "reward": 0.6668527126312256, "reward_std": 0.16180766560137272, "rewards/accuracy_reward/mean": 0.16964285541325808, "rewards/accuracy_reward/std": 0.31550170853734016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 588 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6986607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 971.575927734375, "completions/mean_terminated_length": 858.175048828125, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "epoch": 0.17593906355014563, "grad_norm": 0.06620630621910095, "kl": 0.101318359375, "learning_rate": 1.9575713608048146e-05, "loss": 0.0088, "num_tokens": 300335287.0, "reward": 0.6132812798023224, "reward_std": 0.12465385906398296, "rewards/accuracy_reward/mean": 0.1138392835855484, "rewards/accuracy_reward/std": 0.3186618387699127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 589 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7767857142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 991.5670013427734, "completions/mean_terminated_length": 876.9609375, "completions/min_length": 639.5, "completions/min_terminated_length": 639.5, "epoch": 0.1762377716376671, "grad_norm": 0.050941936671733856, "kl": 0.0894775390625, "learning_rate": 1.957256417793807e-05, "loss": 0.0085, "num_tokens": 300857637.0, "reward": 0.580357164144516, "reward_std": 0.06828013365156949, "rewards/accuracy_reward/mean": 0.08258928684517741, "rewards/accuracy_reward/std": 0.2581148333847523, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 590 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7232142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 992.5, "completions/mean_length": 966.3728179931641, "completions/mean_terminated_length": 808.2276763916016, "completions/min_length": 587.25, "completions/min_terminated_length": 587.25, "epoch": 0.17653647972518857, "grad_norm": 0.09765518456697464, "kl": 0.0919189453125, "learning_rate": 1.956940335732209e-05, "loss": 0.0065, "num_tokens": 301370652.0, "reward": 0.619419664144516, "reward_std": 0.12024092860519886, "rewards/accuracy_reward/mean": 0.12053571408614516, "rewards/accuracy_reward/std": 0.30678585171699524, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 591 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7232142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 974.6585235595703, "completions/mean_terminated_length": 861.0069732666016, "completions/min_length": 664.25, "completions/min_terminated_length": 664.25, "epoch": 0.17683518781271004, "grad_norm": 0.05776679515838623, "kl": 0.0897216796875, "learning_rate": 1.9566231149961302e-05, "loss": 0.0018, "num_tokens": 301878947.0, "reward": 0.6473214626312256, "reward_std": 0.09593360032886267, "rewards/accuracy_reward/mean": 0.14732142654247582, "rewards/accuracy_reward/std": 0.3051318731158972, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.5, "rewards/tag_count_reward/std": 0.0, "step": 592 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7790178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 988.9598541259766, "completions/mean_terminated_length": 874.1621704101562, "completions/min_length": 564.25, "completions/min_terminated_length": 564.25, "epoch": 0.17713389590023149, "grad_norm": 0.06796947121620178, "kl": 0.0924072265625, "learning_rate": 1.9563047559630356e-05, "loss": 0.0048, "num_tokens": 302400497.0, "reward": 0.649553582072258, "reward_std": 0.1557425558567047, "rewards/accuracy_reward/mean": 0.14955356903374195, "rewards/accuracy_reward/std": 0.3442533649504185, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.5, "rewards/tag_count_reward/std": 0.0, "step": 593 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6473214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 968.3348693847656, "completions/mean_terminated_length": 869.2617645263672, "completions/min_length": 515.75, "completions/min_terminated_length": 515.75, "epoch": 0.17743260398775296, "grad_norm": 0.05745833367109299, "kl": 0.0936279296875, "learning_rate": 1.955985259011744e-05, "loss": 0.0012, "num_tokens": 302909687.0, "reward": 0.648995578289032, "reward_std": 0.1287944968789816, "rewards/accuracy_reward/mean": 0.14955357369035482, "rewards/accuracy_reward/std": 0.3422991782426834, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 594 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6763392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 970.779052734375, "completions/mean_terminated_length": 861.2031097412109, "completions/min_length": 517.25, "completions/min_terminated_length": 517.25, "epoch": 0.17773131207527443, "grad_norm": 0.0724555253982544, "kl": 0.1025390625, "learning_rate": 1.9556646245224286e-05, "loss": 0.0057, "num_tokens": 303423412.0, "reward": 0.599888414144516, "reward_std": 0.14073292538523674, "rewards/accuracy_reward/mean": 0.10044643096625805, "rewards/accuracy_reward/std": 0.29379379004240036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 595 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 902.2589569091797, "completions/mean_terminated_length": 795.8549957275391, "completions/min_length": 414.75, "completions/min_terminated_length": 414.75, "epoch": 0.1780300201627959, "grad_norm": 0.0494944304227829, "kl": 0.1021728515625, "learning_rate": 1.9553428528766163e-05, "loss": 0.0019, "num_tokens": 303900712.0, "reward": 0.5915178954601288, "reward_std": 0.07066834531724453, "rewards/accuracy_reward/mean": 0.09151785681024194, "rewards/accuracy_reward/std": 0.2167319767177105, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.5, "rewards/tag_count_reward/std": 0.0, "step": 596 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7522321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 987.7455902099609, "completions/mean_terminated_length": 875.6141357421875, "completions/min_length": 487.5, "completions/min_terminated_length": 487.5, "epoch": 0.17832872825031737, "grad_norm": 0.05480053648352623, "kl": 0.1026611328125, "learning_rate": 1.9550199444571868e-05, "loss": 0.0092, "num_tokens": 304411958.0, "reward": 0.5524553656578064, "reward_std": 0.06929928623139858, "rewards/accuracy_reward/mean": 0.05357142956927419, "rewards/accuracy_reward/std": 0.19716566056013107, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 597 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6540178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 973.7433471679688, "completions/mean_terminated_length": 875.2693023681641, "completions/min_length": 552.75, "completions/min_terminated_length": 552.75, "epoch": 0.17862743633783884, "grad_norm": 0.0744457021355629, "kl": 0.1070556640625, "learning_rate": 1.9546958996483725e-05, "loss": 0.0061, "num_tokens": 304918147.0, "reward": 0.6043526828289032, "reward_std": 0.13503964059054852, "rewards/accuracy_reward/mean": 0.10937500046566129, "rewards/accuracy_reward/std": 0.29292429611086845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 598 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 943.8683624267578, "completions/mean_terminated_length": 838.0286407470703, "completions/min_length": 440.25, "completions/min_terminated_length": 440.25, "epoch": 0.1789261444253603, "grad_norm": 0.06877664476633072, "kl": 0.108154296875, "learning_rate": 1.9543707188357583e-05, "loss": 0.0078, "num_tokens": 305410840.0, "reward": 0.658482164144516, "reward_std": 0.12630531005561352, "rewards/accuracy_reward/mean": 0.1607142868451774, "rewards/accuracy_reward/std": 0.34107325971126556, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 599 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 959.5803833007812, "completions/mean_terminated_length": 833.6050567626953, "completions/min_length": 434.75, "completions/min_terminated_length": 434.75, "epoch": 0.17922485251288178, "grad_norm": 0.08844861388206482, "kl": 0.118896484375, "learning_rate": 1.9540444024062807e-05, "loss": 0.0043, "num_tokens": 305917372.0, "reward": 0.5770089477300644, "reward_std": 0.10812907526269555, "rewards/accuracy_reward/mean": 0.08258928544819355, "rewards/accuracy_reward/std": 0.23133131489157677, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.029416739474982023, "step": 600 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7276785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 974.6049499511719, "completions/mean_terminated_length": 844.5166168212891, "completions/min_length": 474.25, "completions/min_terminated_length": 474.25, "epoch": 0.17952356060040325, "grad_norm": 0.11778488755226135, "kl": 0.12451171875, "learning_rate": 1.953716950748227e-05, "loss": 0.001, "num_tokens": 306425115.0, "reward": 0.5876116305589676, "reward_std": 0.14933771267533302, "rewards/accuracy_reward/mean": 0.09598214132711291, "rewards/accuracy_reward/std": 0.2700103707611561, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 601 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7678571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 988.1540679931641, "completions/mean_terminated_length": 871.4729766845703, "completions/min_length": 539.75, "completions/min_terminated_length": 539.75, "epoch": 0.17982226868792472, "grad_norm": 0.14092765748500824, "kl": 0.1295166015625, "learning_rate": 1.9533883642512363e-05, "loss": -0.0027, "num_tokens": 306951296.0, "reward": 0.6328125298023224, "reward_std": 0.19951780512928963, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.33531374484300613, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.056248647160828114, "step": 602 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 963.1920013427734, "completions/mean_terminated_length": 864.4721984863281, "completions/min_length": 470.0, "completions/min_terminated_length": 470.0, "epoch": 0.1801209767754462, "grad_norm": 0.09206223487854004, "kl": 0.148193359375, "learning_rate": 1.9530586433062973e-05, "loss": -0.0083, "num_tokens": 307448630.0, "reward": 0.6389509290456772, "reward_std": 0.11889695003628731, "rewards/accuracy_reward/mean": 0.14732143143191934, "rewards/accuracy_reward/std": 0.32089732214808464, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 603 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7522321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 975.8370819091797, "completions/mean_terminated_length": 829.0075836181641, "completions/min_length": 475.5, "completions/min_terminated_length": 475.5, "epoch": 0.18041968486296767, "grad_norm": 0.14098216593265533, "kl": 0.19677734375, "learning_rate": 1.9527277883057484e-05, "loss": -0.0181, "num_tokens": 307955613.0, "reward": 0.6138392984867096, "reward_std": 0.17416252382099628, "rewards/accuracy_reward/mean": 0.12499999674037099, "rewards/accuracy_reward/std": 0.3077162466943264, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051574116572737694, "step": 604 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7232142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 974.2567443847656, "completions/mean_terminated_length": 853.6355438232422, "completions/min_length": 524.75, "completions/min_terminated_length": 524.75, "epoch": 0.18071839295048914, "grad_norm": 0.16884101927280426, "kl": 0.28564453125, "learning_rate": 1.9523957996432785e-05, "loss": 0.0039, "num_tokens": 308462160.0, "reward": 0.636160746216774, "reward_std": 0.132711386308074, "rewards/accuracy_reward/mean": 0.14955357206054032, "rewards/accuracy_reward/std": 0.32091367803514004, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05471411347389221, "step": 605 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7589285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 967.3281555175781, "completions/mean_terminated_length": 786.9302368164062, "completions/min_length": 394.25, "completions/min_terminated_length": 394.25, "epoch": 0.1810171010380106, "grad_norm": 0.22835077345371246, "kl": 0.546875, "learning_rate": 1.9520626777139243e-05, "loss": -0.0482, "num_tokens": 308967747.0, "reward": 0.5368303805589676, "reward_std": 0.14790860377252102, "rewards/accuracy_reward/mean": 0.06026785750873387, "rewards/accuracy_reward/std": 0.22483053617179394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4765625, "rewards/tag_count_reward/std": 0.07286412827670574, "step": 606 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7857142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 969.5111999511719, "completions/mean_terminated_length": 754.3497467041016, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.18131580912553208, "grad_norm": 0.4612884521484375, "kl": 0.8388671875, "learning_rate": 1.9517284229140718e-05, "loss": -0.0395, "num_tokens": 309478760.0, "reward": 0.5390625149011612, "reward_std": 0.1503484956920147, "rewards/accuracy_reward/mean": 0.06696428591385484, "rewards/accuracy_reward/std": 0.23752513900399208, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.472098208963871, "rewards/tag_count_reward/std": 0.07866431027650833, "step": 607 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.921875, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.5, "completions/mean_length": 1009.6317291259766, "completions/mean_terminated_length": 844.1541748046875, "completions/min_length": 592.5, "completions/min_terminated_length": 592.5, "epoch": 0.18161451721305355, "grad_norm": 0.21325072646141052, "kl": 0.41748046875, "learning_rate": 1.9513930356414546e-05, "loss": 0.0039, "num_tokens": 310010083.0, "reward": 0.5959821790456772, "reward_std": 0.2508351653814316, "rewards/accuracy_reward/mean": 0.1450892835855484, "rewards/accuracy_reward/std": 0.3464914485812187, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4508928507566452, "rewards/tag_count_reward/std": 0.09875807724893093, "step": 608 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 986.25, "completions/mean_length": 1016.1562805175781, "completions/mean_terminated_length": 840.1319580078125, "completions/min_length": 605.75, "completions/min_terminated_length": 605.75, "epoch": 0.18191322530057502, "grad_norm": 0.2741043269634247, "kl": 0.4521484375, "learning_rate": 1.9510565162951538e-05, "loss": 0.0119, "num_tokens": 310534137.0, "reward": 0.4972098395228386, "reward_std": 0.22610897198319435, "rewards/accuracy_reward/mean": 0.06250000046566129, "rewards/accuracy_reward/std": 0.23696717619895935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4347098246216774, "rewards/tag_count_reward/std": 0.11006903648376465, "step": 609 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9263392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 1014.5536041259766, "completions/mean_terminated_length": 902.0243072509766, "completions/min_length": 699.25, "completions/min_terminated_length": 699.25, "epoch": 0.1822119333880965, "grad_norm": 0.3871587812900543, "kl": 0.38818359375, "learning_rate": 1.950718865275598e-05, "loss": 0.0158, "num_tokens": 311058849.0, "reward": 0.5887277126312256, "reward_std": 0.23104705661535263, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.35125770419836044, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.44140625, "rewards/tag_count_reward/std": 0.10613344982266426, "step": 610 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 992.982177734375, "completions/mean_terminated_length": 853.9379425048828, "completions/min_length": 639.5, "completions/min_terminated_length": 639.5, "epoch": 0.18251064147561796, "grad_norm": 0.2515324354171753, "kl": 1.0859375, "learning_rate": 1.9503800829845613e-05, "loss": 0.0385, "num_tokens": 311569625.0, "reward": 0.6054687798023224, "reward_std": 0.19907857105135918, "rewards/accuracy_reward/mean": 0.14657738176174462, "rewards/accuracy_reward/std": 0.3194946311414242, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4670758917927742, "rewards/tag_count_reward/std": 0.08456084132194519, "step": 611 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 1003.5000457763672, "completions/mean_terminated_length": 844.6205444335938, "completions/min_length": 506.75, "completions/min_terminated_length": 506.75, "epoch": 0.18280934956313943, "grad_norm": 0.8725905418395996, "kl": 2.029296875, "learning_rate": 1.9500401698251657e-05, "loss": 0.0546, "num_tokens": 312088073.0, "reward": 0.556919664144516, "reward_std": 0.1330952262505889, "rewards/accuracy_reward/mean": 0.07589285681024194, "rewards/accuracy_reward/std": 0.1986122652888298, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06561115104705095, "step": 612 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 1007.9665679931641, "completions/mean_terminated_length": 889.3198699951172, "completions/min_length": 678.75, "completions/min_terminated_length": 678.75, "epoch": 0.1831080576506609, "grad_norm": 0.5184608697891235, "kl": 0.9794921875, "learning_rate": 1.949699126201877e-05, "loss": 0.0441, "num_tokens": 312612218.0, "reward": 0.5189732387661934, "reward_std": 0.1481250636279583, "rewards/accuracy_reward/mean": 0.060639883391559124, "rewards/accuracy_reward/std": 0.21717893332242966, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4631696417927742, "rewards/tag_count_reward/std": 0.09019191935658455, "step": 613 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 1009.1361999511719, "completions/mean_terminated_length": 862.5119323730469, "completions/min_length": 618.5, "completions/min_terminated_length": 618.5, "epoch": 0.18340676573818235, "grad_norm": 0.3597496449947357, "kl": 0.8662109375, "learning_rate": 1.9493569525205077e-05, "loss": 0.0333, "num_tokens": 313135607.0, "reward": 0.5323660969734192, "reward_std": 0.16802511177957058, "rewards/accuracy_reward/mean": 0.06919642724096775, "rewards/accuracy_reward/std": 0.21212364733219147, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4631696343421936, "rewards/tag_count_reward/std": 0.08899068832397461, "step": 614 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 1006.0893249511719, "completions/mean_terminated_length": 830.53662109375, "completions/min_length": 443.5, "completions/min_terminated_length": 443.5, "epoch": 0.18370547382570382, "grad_norm": 0.30677902698516846, "kl": 1.212890625, "learning_rate": 1.9490136491882143e-05, "loss": 0.0284, "num_tokens": 313650431.0, "reward": 0.5345982313156128, "reward_std": 0.17627441138029099, "rewards/accuracy_reward/mean": 0.06473214295692742, "rewards/accuracy_reward/std": 0.21900047734379768, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4698660671710968, "rewards/tag_count_reward/std": 0.08125760965049267, "step": 615 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8549107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 1002.497802734375, "completions/mean_terminated_length": 874.7750549316406, "completions/min_length": 667.75, "completions/min_terminated_length": 667.75, "epoch": 0.1840041819132253, "grad_norm": 0.7565816044807434, "kl": 1.5, "learning_rate": 1.9486692166134964e-05, "loss": 0.0508, "num_tokens": 314171886.0, "reward": 0.5948660969734192, "reward_std": 0.1948801688849926, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.31716566532850266, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.06951318681240082, "step": 616 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 1010.6786193847656, "completions/mean_terminated_length": 892.8628540039062, "completions/min_length": 645.0, "completions/min_terminated_length": 645.0, "epoch": 0.18430289000074676, "grad_norm": 0.5537185668945312, "kl": 0.47802734375, "learning_rate": 1.9483236552061996e-05, "loss": 0.0173, "num_tokens": 314693598.0, "reward": 0.538504496216774, "reward_std": 0.15636065043509007, "rewards/accuracy_reward/mean": 0.08258928474970162, "rewards/accuracy_reward/std": 0.25298800505697727, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4559151753783226, "rewards/tag_count_reward/std": 0.09450318664312363, "step": 617 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 1016.6808319091797, "completions/mean_terminated_length": 923.639892578125, "completions/min_length": 751.5, "completions/min_terminated_length": 751.5, "epoch": 0.18460159808826823, "grad_norm": 0.45061665773391724, "kl": 0.44775390625, "learning_rate": 1.9479769653775107e-05, "loss": 0.0183, "num_tokens": 315222911.0, "reward": 0.5457589477300644, "reward_std": 0.14368786104023457, "rewards/accuracy_reward/mean": 0.07589285518042743, "rewards/accuracy_reward/std": 0.2331551667302847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4698660671710968, "rewards/tag_count_reward/std": 0.07989407982677221, "step": 618 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 994.529052734375, "completions/mean_terminated_length": 870.3083343505859, "completions/min_length": 681.25, "completions/min_terminated_length": 681.25, "epoch": 0.1849003061757897, "grad_norm": 0.15362980961799622, "kl": 0.431640625, "learning_rate": 1.94762914753996e-05, "loss": 0.0113, "num_tokens": 315750636.0, "reward": 0.650669664144516, "reward_std": 0.13423519674688578, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.35745642334222794, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 619 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 987.7076416015625, "completions/mean_terminated_length": 857.2711944580078, "completions/min_length": 556.25, "completions/min_terminated_length": 556.25, "epoch": 0.18519901426331117, "grad_norm": 0.4208965301513672, "kl": 0.6494140625, "learning_rate": 1.9472802021074203e-05, "loss": 0.0051, "num_tokens": 316261353.0, "reward": 0.5496652126312256, "reward_std": 0.07702670246362686, "rewards/accuracy_reward/mean": 0.06250000046566129, "rewards/accuracy_reward/std": 0.19397937878966331, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05343634821474552, "step": 620 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 1004.8906707763672, "completions/mean_terminated_length": 911.8556823730469, "completions/min_length": 714.75, "completions/min_terminated_length": 714.75, "epoch": 0.18549772235083264, "grad_norm": 0.34441429376602173, "kl": 0.45361328125, "learning_rate": 1.946930129495106e-05, "loss": 0.0138, "num_tokens": 316787752.0, "reward": 0.6936384290456772, "reward_std": 0.1414129752665758, "rewards/accuracy_reward/mean": 0.19866072107106447, "rewards/accuracy_reward/std": 0.3594658151268959, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.02920706057921052, "step": 621 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7209821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 980.5737152099609, "completions/mean_terminated_length": 866.8843536376953, "completions/min_length": 603.75, "completions/min_terminated_length": 603.75, "epoch": 0.18579643043835412, "grad_norm": 0.2276764065027237, "kl": 0.2666015625, "learning_rate": 1.946578930119572e-05, "loss": 0.0138, "num_tokens": 317299049.0, "reward": 0.7555803954601288, "reward_std": 0.20502671971917152, "rewards/accuracy_reward/mean": 0.2633928582072258, "rewards/accuracy_reward/std": 0.4375237599015236, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.041961644776165485, "step": 622 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.75, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 973.9777221679688, "completions/mean_terminated_length": 843.2758636474609, "completions/min_length": 578.25, "completions/min_terminated_length": 578.25, "epoch": 0.1860951385258756, "grad_norm": 0.1080431193113327, "kl": 0.19677734375, "learning_rate": 1.9462266043987148e-05, "loss": 0.0117, "num_tokens": 317809391.0, "reward": 0.7466518133878708, "reward_std": 0.22497126087546349, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.40454409271478653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517761349678, "rewards/tag_count_reward/std": 0.024942624382674694, "step": 623 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.765625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 980.8772735595703, "completions/mean_terminated_length": 851.7853088378906, "completions/min_length": 565.5, "completions/min_terminated_length": 565.5, "epoch": 0.18639384661339706, "grad_norm": 0.20037460327148438, "kl": 0.171142578125, "learning_rate": 1.9458731527517712e-05, "loss": 0.0046, "num_tokens": 318323880.0, "reward": 0.7438616454601288, "reward_std": 0.16273482888936996, "rewards/accuracy_reward/mean": 0.24999999813735485, "rewards/accuracy_reward/std": 0.41519831866025925, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.037908039055764675, "step": 624 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 1000.841552734375, "completions/mean_terminated_length": 898.0926818847656, "completions/min_length": 643.0, "completions/min_terminated_length": 643.0, "epoch": 0.18669255470091853, "grad_norm": 0.2288956642150879, "kl": 0.185546875, "learning_rate": 1.945518575599317e-05, "loss": 0.0071, "num_tokens": 318845553.0, "reward": 0.6450893133878708, "reward_std": 0.12563525512814522, "rewards/accuracy_reward/mean": 0.15178571455180645, "rewards/accuracy_reward/std": 0.3502853736281395, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.038836000952869654, "step": 625 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7924107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 995.2969055175781, "completions/mean_terminated_length": 895.2609100341797, "completions/min_length": 667.5, "completions/min_terminated_length": 667.5, "epoch": 0.18699126278844, "grad_norm": 0.10168388485908508, "kl": 0.169921875, "learning_rate": 1.945162873363268e-05, "loss": 0.0072, "num_tokens": 319371510.0, "reward": 0.6484375298023224, "reward_std": 0.09946922701783478, "rewards/accuracy_reward/mean": 0.14955357275903225, "rewards/accuracy_reward/std": 0.2877371460199356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 626 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7991071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 983.2121124267578, "completions/mean_terminated_length": 828.8689880371094, "completions/min_length": 544.25, "completions/min_terminated_length": 544.25, "epoch": 0.18728997087596147, "grad_norm": 0.08517933636903763, "kl": 0.171875, "learning_rate": 1.944806046466878e-05, "loss": 0.009, "num_tokens": 319880149.0, "reward": 0.689732164144516, "reward_std": 0.16894518211483955, "rewards/accuracy_reward/mean": 0.1982886902987957, "rewards/accuracy_reward/std": 0.39119812101125717, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 627 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7633928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 977.997802734375, "completions/mean_terminated_length": 828.7900085449219, "completions/min_length": 566.5, "completions/min_terminated_length": 566.5, "epoch": 0.18758867896348294, "grad_norm": 0.07910898327827454, "kl": 0.16552734375, "learning_rate": 1.9444480953347402e-05, "loss": 0.0058, "num_tokens": 320385716.0, "reward": 0.6266741305589676, "reward_std": 0.11339907837100327, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.2721429839730263, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 628 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8303571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 1004.9062957763672, "completions/mean_terminated_length": 930.7014617919922, "completions/min_length": 763.0, "completions/min_terminated_length": 763.0, "epoch": 0.1878873870510044, "grad_norm": 0.0924113318324089, "kl": 0.1904296875, "learning_rate": 1.9440890203927846e-05, "loss": 0.0051, "num_tokens": 320906762.0, "reward": 0.6484375298023224, "reward_std": 0.13775707222521305, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.33691973239183426, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 629 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 998.0670013427734, "completions/mean_terminated_length": 896.3126373291016, "completions/min_length": 717.25, "completions/min_terminated_length": 717.25, "epoch": 0.18818609513852588, "grad_norm": 0.07805649936199188, "kl": 0.175537109375, "learning_rate": 1.943728822068278e-05, "loss": 0.0073, "num_tokens": 321430168.0, "reward": 0.6802455633878708, "reward_std": 0.11541508138179779, "rewards/accuracy_reward/mean": 0.1808035708963871, "rewards/accuracy_reward/std": 0.3851044923067093, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 630 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7142857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 958.6741638183594, "completions/mean_terminated_length": 781.587158203125, "completions/min_length": 519.0, "completions/min_terminated_length": 519.0, "epoch": 0.18848480322604735, "grad_norm": 0.07904264330863953, "kl": 0.17822265625, "learning_rate": 1.9433675007898255e-05, "loss": 0.009, "num_tokens": 321922278.0, "reward": 0.6305803805589676, "reward_std": 0.11502526514232159, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.3081366531550884, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 631 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 940.9219207763672, "completions/mean_terminated_length": 797.0545959472656, "completions/min_length": 485.5, "completions/min_terminated_length": 485.5, "epoch": 0.18878351131356882, "grad_norm": 0.13140523433685303, "kl": 0.192626953125, "learning_rate": 1.943005056987367e-05, "loss": 0.0131, "num_tokens": 322416915.0, "reward": 0.737723246216774, "reward_std": 0.18510113656520844, "rewards/accuracy_reward/mean": 0.23883928544819355, "rewards/accuracy_reward/std": 0.3930158503353596, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 632 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 990.3683319091797, "completions/mean_terminated_length": 861.3432769775391, "completions/min_length": 576.25, "completions/min_terminated_length": 576.25, "epoch": 0.1890822194010903, "grad_norm": 0.1107010766863823, "kl": 0.239501953125, "learning_rate": 1.9426414910921785e-05, "loss": 0.0098, "num_tokens": 322935320.0, "reward": 0.6367187649011612, "reward_std": 0.08604444935917854, "rewards/accuracy_reward/mean": 0.1383928540162742, "rewards/accuracy_reward/std": 0.3032829575240612, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 633 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7991071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 977.7411193847656, "completions/mean_terminated_length": 814.4351806640625, "completions/min_length": 521.75, "completions/min_terminated_length": 521.75, "epoch": 0.18938092748861177, "grad_norm": 0.07113903015851974, "kl": 0.186767578125, "learning_rate": 1.9422768035368717e-05, "loss": 0.0018, "num_tokens": 323449620.0, "reward": 0.6629464477300644, "reward_std": 0.06859304127283394, "rewards/accuracy_reward/mean": 0.1651785671710968, "rewards/accuracy_reward/std": 0.305410198867321, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 634 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 1009.8504943847656, "completions/mean_terminated_length": 920.7836761474609, "completions/min_length": 772.25, "completions/min_terminated_length": 772.25, "epoch": 0.18967963557613324, "grad_norm": 0.10217278450727463, "kl": 0.25244140625, "learning_rate": 1.9419109947553925e-05, "loss": 0.009, "num_tokens": 323967009.0, "reward": 0.5937500298023224, "reward_std": 0.13461412861943245, "rewards/accuracy_reward/mean": 0.09598214225843549, "rewards/accuracy_reward/std": 0.27069151401519775, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 635 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 957.75, "completions/mean_length": 1011.0714569091797, "completions/mean_terminated_length": 879.0562591552734, "completions/min_length": 672.0, "completions/min_terminated_length": 672.0, "epoch": 0.18997834366365468, "grad_norm": 0.10193317383527756, "kl": 0.2509765625, "learning_rate": 1.941544065183021e-05, "loss": 0.0098, "num_tokens": 324491265.0, "reward": 0.6378348618745804, "reward_std": 0.1531983334571123, "rewards/accuracy_reward/mean": 0.14062500116415322, "rewards/accuracy_reward/std": 0.3144758902490139, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 636 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 998.7232513427734, "completions/mean_terminated_length": 861.0985565185547, "completions/min_length": 626.0, "completions/min_terminated_length": 626.0, "epoch": 0.19027705175117615, "grad_norm": 0.08560260385274887, "kl": 0.26708984375, "learning_rate": 1.941176015256371e-05, "loss": 0.0099, "num_tokens": 325016757.0, "reward": 0.6171875298023224, "reward_std": 0.10386104229837656, "rewards/accuracy_reward/mean": 0.12053571408614516, "rewards/accuracy_reward/std": 0.30809371173381805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.023462072014808655, "step": 637 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.5, "completions/mean_length": 1008.9420013427734, "completions/mean_terminated_length": 883.9024047851562, "completions/min_length": 706.5, "completions/min_terminated_length": 706.5, "epoch": 0.19057575983869762, "grad_norm": 0.0926973819732666, "kl": 0.276123046875, "learning_rate": 1.940806845413389e-05, "loss": 0.0107, "num_tokens": 325540283.0, "reward": 0.5786830484867096, "reward_std": 0.1142335869371891, "rewards/accuracy_reward/mean": 0.08258928591385484, "rewards/accuracy_reward/std": 0.2534317225217819, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 638 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8950892857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.75, "completions/mean_length": 1006.8147735595703, "completions/mean_terminated_length": 891.1833953857422, "completions/min_length": 707.25, "completions/min_terminated_length": 707.25, "epoch": 0.1908744679262191, "grad_norm": 0.09755261987447739, "kl": 0.33642578125, "learning_rate": 1.940436556093355e-05, "loss": 0.009, "num_tokens": 326057544.0, "reward": 0.5379464477300644, "reward_std": 0.1045578233897686, "rewards/accuracy_reward/mean": 0.04464285704307258, "rewards/accuracy_reward/std": 0.18391988426446915, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 639 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 742.25, "completions/mean_length": 1017.7544860839844, "completions/mean_terminated_length": 677.7777862548828, "completions/min_length": 881.25, "completions/min_terminated_length": 625.25, "epoch": 0.19117317601374056, "grad_norm": 0.2621022164821625, "kl": 0.35693359375, "learning_rate": 1.9400651477368804e-05, "loss": 0.0152, "num_tokens": 326583562.0, "reward": 0.7287946790456772, "reward_std": 0.18877706117928028, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.39958375692367554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.060212516225874424, "step": 640 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.96875, "completions/max_length": 1024.0, "completions/max_terminated_length": 951.0, "completions/mean_length": 1021.1072082519531, "completions/mean_terminated_length": 914.2750091552734, "completions/min_length": 872.25, "completions/min_terminated_length": 872.25, "epoch": 0.19147188410126204, "grad_norm": 0.11038003861904144, "kl": 0.32763671875, "learning_rate": 1.9396926207859085e-05, "loss": 0.0124, "num_tokens": 327110074.0, "reward": 0.6132812798023224, "reward_std": 0.14085173048079014, "rewards/accuracy_reward/mean": 0.12276785727590322, "rewards/accuracy_reward/std": 0.30876875668764114, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.045961628668010235, "step": 641 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.25, "completions/mean_length": 1018.5893249511719, "completions/mean_terminated_length": 944.6083679199219, "completions/min_length": 830.5, "completions/min_terminated_length": 830.5, "epoch": 0.1917705921887835, "grad_norm": 0.11978872865438461, "kl": 0.40234375, "learning_rate": 1.939318975683713e-05, "loss": 0.0159, "num_tokens": 327641746.0, "reward": 0.6300223618745804, "reward_std": 0.17143219336867332, "rewards/accuracy_reward/mean": 0.13839286006987095, "rewards/accuracy_reward/std": 0.33516503870487213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 642 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8816964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.5, "completions/mean_length": 1000.7924499511719, "completions/mean_terminated_length": 875.9773864746094, "completions/min_length": 663.25, "completions/min_terminated_length": 663.25, "epoch": 0.19206930027630498, "grad_norm": 0.20935027301311493, "kl": 0.54443359375, "learning_rate": 1.9389442128748994e-05, "loss": 0.0204, "num_tokens": 328162885.0, "reward": 0.5881696566939354, "reward_std": 0.08990370109677315, "rewards/accuracy_reward/mean": 0.09598213993012905, "rewards/accuracy_reward/std": 0.24942436069250107, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.03723818250000477, "step": 643 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 995.5625457763672, "completions/mean_terminated_length": 844.3677673339844, "completions/min_length": 560.25, "completions/min_terminated_length": 560.25, "epoch": 0.19236800836382645, "grad_norm": 0.1510201245546341, "kl": 0.59814453125, "learning_rate": 1.938568332805402e-05, "loss": 0.0217, "num_tokens": 328685969.0, "reward": 0.5758928805589676, "reward_std": 0.13886883109807968, "rewards/accuracy_reward/mean": 0.09635416697710752, "rewards/accuracy_reward/std": 0.2912408709526062, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.06006421521306038, "step": 644 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 1012.6964874267578, "completions/mean_terminated_length": 888.4785766601562, "completions/min_length": 730.25, "completions/min_terminated_length": 730.25, "epoch": 0.19266671645134792, "grad_norm": 0.1385868340730667, "kl": 0.44775390625, "learning_rate": 1.9381913359224844e-05, "loss": 0.0207, "num_tokens": 329216937.0, "reward": 0.6350446790456772, "reward_std": 0.18934544920921326, "rewards/accuracy_reward/mean": 0.14732142817229033, "rewards/accuracy_reward/std": 0.33970701694488525, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05248269159346819, "step": 645 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 1019.6830902099609, "completions/mean_terminated_length": 917.1875152587891, "completions/min_length": 827.25, "completions/min_terminated_length": 827.25, "epoch": 0.1929654245388694, "grad_norm": 0.21043656766414642, "kl": 0.4140625, "learning_rate": 1.93781322267474e-05, "loss": 0.0162, "num_tokens": 329741547.0, "reward": 0.6188616454601288, "reward_std": 0.12658806703984737, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.3170383982360363, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 646 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.5, "completions/mean_length": 1006.669677734375, "completions/mean_terminated_length": 882.4797515869141, "completions/min_length": 712.0, "completions/min_terminated_length": 712.0, "epoch": 0.19326413262639086, "grad_norm": 0.37225019931793213, "kl": 0.5615234375, "learning_rate": 1.93743399351209e-05, "loss": 0.0184, "num_tokens": 330273447.0, "reward": 0.6462053805589676, "reward_std": 0.16860496997833252, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.34162159636616707, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 647 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 1002.5357513427734, "completions/mean_terminated_length": 901.6857147216797, "completions/min_length": 727.5, "completions/min_terminated_length": 727.5, "epoch": 0.19356284071391233, "grad_norm": 0.13363845646381378, "kl": 0.4619140625, "learning_rate": 1.9370536488857837e-05, "loss": 0.0182, "num_tokens": 330800407.0, "reward": 0.6372768059372902, "reward_std": 0.11612748820334673, "rewards/accuracy_reward/mean": 0.1473214253783226, "rewards/accuracy_reward/std": 0.296484611928463, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04702208936214447, "step": 648 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.25, "completions/mean_length": 1008.1094207763672, "completions/mean_terminated_length": 882.9031829833984, "completions/min_length": 711.0, "completions/min_terminated_length": 711.0, "epoch": 0.1938615488014338, "grad_norm": 0.2253638207912445, "kl": 0.501953125, "learning_rate": 1.9366721892483976e-05, "loss": 0.0216, "num_tokens": 331324744.0, "reward": 0.619419664144516, "reward_std": 0.20762836560606956, "rewards/accuracy_reward/mean": 0.14062499720603228, "rewards/accuracy_reward/std": 0.32279664278030396, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.0689700935035944, "step": 649 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 1006.9777221679688, "completions/mean_terminated_length": 900.7152862548828, "completions/min_length": 711.0, "completions/min_terminated_length": 711.0, "epoch": 0.19416025688895527, "grad_norm": 0.21430176496505737, "kl": 0.5908203125, "learning_rate": 1.9362896150538354e-05, "loss": 0.0235, "num_tokens": 331840926.0, "reward": 0.618861623108387, "reward_std": 0.1353129642084241, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.2891194596886635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4760044664144516, "rewards/tag_count_reward/std": 0.0730813667178154, "step": 650 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.9040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 1012.1696929931641, "completions/mean_terminated_length": 915.2686462402344, "completions/min_length": 707.25, "completions/min_terminated_length": 707.25, "epoch": 0.19445896497647674, "grad_norm": 0.2213943749666214, "kl": 0.6533203125, "learning_rate": 1.935905926757326e-05, "loss": 0.0249, "num_tokens": 332363402.0, "reward": 0.5345982313156128, "reward_std": 0.12416384927928448, "rewards/accuracy_reward/mean": 0.05357142863795161, "rewards/accuracy_reward/std": 0.18438423797488213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267761349678, "rewards/tag_count_reward/std": 0.06639691535383463, "step": 651 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 1001.4152221679688, "completions/mean_terminated_length": 873.6704711914062, "completions/min_length": 546.5, "completions/min_terminated_length": 546.5, "epoch": 0.19475767306399822, "grad_norm": 0.30093157291412354, "kl": 0.9150390625, "learning_rate": 1.9355211248154247e-05, "loss": 0.0267, "num_tokens": 332888820.0, "reward": 0.6406250298023224, "reward_std": 0.17732481472194195, "rewards/accuracy_reward/mean": 0.15624999906867743, "rewards/accuracy_reward/std": 0.3482544273138046, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.06053628120571375, "step": 652 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 986.0759429931641, "completions/mean_terminated_length": 823.6263427734375, "completions/min_length": 569.75, "completions/min_terminated_length": 569.75, "epoch": 0.1950563811515197, "grad_norm": 0.19051127135753632, "kl": 0.796875, "learning_rate": 1.935135209686012e-05, "loss": 0.0378, "num_tokens": 333401222.0, "reward": 0.632254496216774, "reward_std": 0.1346572171896696, "rewards/accuracy_reward/mean": 0.14508928661234677, "rewards/accuracy_reward/std": 0.31002812646329403, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.053371951915323734, "step": 653 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7299107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 977.1987152099609, "completions/mean_terminated_length": 851.9048156738281, "completions/min_length": 612.0, "completions/min_terminated_length": 612.0, "epoch": 0.19535508923904116, "grad_norm": 0.18995891511440277, "kl": 0.9404296875, "learning_rate": 1.9347481818282927e-05, "loss": 0.0379, "num_tokens": 333911135.0, "reward": 0.6177455708384514, "reward_std": 0.15033418498933315, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.27965374290943146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057406721636652946, "step": 654 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 1002.6808471679688, "completions/mean_terminated_length": 878.2664642333984, "completions/min_length": 606.25, "completions/min_terminated_length": 606.25, "epoch": 0.19565379732656263, "grad_norm": 0.31126970052719116, "kl": 1.326171875, "learning_rate": 1.9343600417027956e-05, "loss": 0.0418, "num_tokens": 334435056.0, "reward": 0.493861623108387, "reward_std": 0.10428975149989128, "rewards/accuracy_reward/mean": 0.022321428637951612, "rewards/accuracy_reward/std": 0.10133291408419609, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4715401753783226, "rewards/tag_count_reward/std": 0.07900950685143471, "step": 655 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7254464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 978.857177734375, "completions/mean_terminated_length": 862.2076568603516, "completions/min_length": 420.5, "completions/min_terminated_length": 420.5, "epoch": 0.1959525054140841, "grad_norm": 0.24927885830402374, "kl": 1.1162109375, "learning_rate": 1.9339707897713737e-05, "loss": 0.0259, "num_tokens": 334945968.0, "reward": 0.6177455633878708, "reward_std": 0.17440848611295223, "rewards/accuracy_reward/mean": 0.14508928707800806, "rewards/accuracy_reward/std": 0.2769778463989496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4726562574505806, "rewards/tag_count_reward/std": 0.07666278816759586, "step": 656 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7544642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 978.1250457763672, "completions/mean_terminated_length": 846.2439727783203, "completions/min_length": 512.5, "completions/min_terminated_length": 512.5, "epoch": 0.19625121350160554, "grad_norm": 0.33999359607696533, "kl": 0.849609375, "learning_rate": 1.9335804264972018e-05, "loss": 0.0308, "num_tokens": 335451720.0, "reward": 0.7059152275323868, "reward_std": 0.1865447573363781, "rewards/accuracy_reward/mean": 0.22544643096625805, "rewards/accuracy_reward/std": 0.41136304289102554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48046875, "rewards/tag_count_reward/std": 0.06687816977500916, "step": 657 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8303571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.5, "completions/mean_length": 988.7678985595703, "completions/mean_terminated_length": 818.3645172119141, "completions/min_length": 315.25, "completions/min_terminated_length": 315.25, "epoch": 0.196549921589127, "grad_norm": 0.4294265806674957, "kl": 1.6875, "learning_rate": 1.933188952344778e-05, "loss": 0.0369, "num_tokens": 335975856.0, "reward": 0.5033482387661934, "reward_std": 0.1390574909746647, "rewards/accuracy_reward/mean": 0.03645833348855376, "rewards/accuracy_reward/std": 0.15602240338921547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4676339253783226, "rewards/tag_count_reward/std": 0.08369126729667187, "step": 658 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6450892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 951.4754791259766, "completions/mean_terminated_length": 817.7590179443359, "completions/min_length": 152.0, "completions/min_terminated_length": 152.0, "epoch": 0.19684862967664848, "grad_norm": 0.29914602637290955, "kl": 1.3349609375, "learning_rate": 1.9327963677799224e-05, "loss": 0.0164, "num_tokens": 336472997.0, "reward": 0.5122767984867096, "reward_std": 0.09770095255225897, "rewards/accuracy_reward/mean": 0.031250000931322575, "rewards/accuracy_reward/std": 0.14576534926891327, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.481026791036129, "rewards/tag_count_reward/std": 0.06513937469571829, "step": 659 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 948.5513916015625, "completions/mean_terminated_length": 806.63818359375, "completions/min_length": 220.5, "completions/min_terminated_length": 220.5, "epoch": 0.19714733776416996, "grad_norm": 0.905705988407135, "kl": 1.447265625, "learning_rate": 1.9324026732697754e-05, "loss": -0.01, "num_tokens": 336974636.0, "reward": 0.5725446790456772, "reward_std": 0.17560767009854317, "rewards/accuracy_reward/mean": 0.08705357136204839, "rewards/accuracy_reward/std": 0.2633744925260544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05743632931262255, "step": 660 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 963.4196929931641, "completions/mean_terminated_length": 813.0186920166016, "completions/min_length": 306.25, "completions/min_terminated_length": 306.25, "epoch": 0.19744604585169143, "grad_norm": 0.19988380372524261, "kl": 0.67578125, "learning_rate": 1.932007869282799e-05, "loss": 0.002, "num_tokens": 337476248.0, "reward": 0.6417410969734192, "reward_std": 0.16519339755177498, "rewards/accuracy_reward/mean": 0.1581101231276989, "rewards/accuracy_reward/std": 0.34337200224399567, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05590797308832407, "step": 661 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 1009.4531555175781, "completions/mean_terminated_length": 908.8793029785156, "completions/min_length": 558.25, "completions/min_terminated_length": 558.25, "epoch": 0.1977447539392129, "grad_norm": 0.4139058291912079, "kl": 0.284423828125, "learning_rate": 1.9316119562887744e-05, "loss": 0.0097, "num_tokens": 338003955.0, "reward": 0.5898437798023224, "reward_std": 0.1510687656700611, "rewards/accuracy_reward/mean": 0.10714285727590322, "rewards/accuracy_reward/std": 0.3041174113750458, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06174672581255436, "step": 662 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 1010.3817443847656, "completions/mean_terminated_length": 914.0805511474609, "completions/min_length": 715.0, "completions/min_terminated_length": 715.0, "epoch": 0.19804346202673437, "grad_norm": 0.35219070315361023, "kl": 0.18896484375, "learning_rate": 1.9312149347588035e-05, "loss": 0.0098, "num_tokens": 338531966.0, "reward": 0.6529018133878708, "reward_std": 0.18404356762766838, "rewards/accuracy_reward/mean": 0.16517857648432255, "rewards/accuracy_reward/std": 0.3573502078652382, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05409308057278395, "step": 663 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 1001.6295166015625, "completions/mean_terminated_length": 876.4286346435547, "completions/min_length": 621.0, "completions/min_terminated_length": 621.0, "epoch": 0.19834217011425584, "grad_norm": 0.385009229183197, "kl": 0.177734375, "learning_rate": 1.9308168051653077e-05, "loss": 0.0046, "num_tokens": 339058280.0, "reward": 0.575334832072258, "reward_std": 0.12158092041499913, "rewards/accuracy_reward/mean": 0.08705357275903225, "rewards/accuracy_reward/std": 0.2402816042304039, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04914125660434365, "step": 664 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8258928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 1001.6518249511719, "completions/mean_terminated_length": 896.9073486328125, "completions/min_length": 440.75, "completions/min_terminated_length": 440.75, "epoch": 0.1986408782017773, "grad_norm": 0.17307746410369873, "kl": 0.25537109375, "learning_rate": 1.9304175679820247e-05, "loss": 0.0013, "num_tokens": 339587932.0, "reward": 0.6227678954601288, "reward_std": 0.11006946349516511, "rewards/accuracy_reward/mean": 0.1294642873108387, "rewards/accuracy_reward/std": 0.25585051253437996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.04216536181047559, "step": 665 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 999.1138916015625, "completions/mean_terminated_length": 905.6520385742188, "completions/min_length": 685.0, "completions/min_terminated_length": 685.0, "epoch": 0.19893958628929878, "grad_norm": 0.1246715858578682, "kl": 0.384521484375, "learning_rate": 1.930017223684012e-05, "loss": 0.0115, "num_tokens": 340105199.0, "reward": 0.7114955633878708, "reward_std": 0.2235245630145073, "rewards/accuracy_reward/mean": 0.2165178544819355, "rewards/accuracy_reward/std": 0.4103814959526062, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 666 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8303571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 1002.0447082519531, "completions/mean_terminated_length": 909.8811492919922, "completions/min_length": 715.25, "completions/min_terminated_length": 715.25, "epoch": 0.19923829437682025, "grad_norm": 0.15880906581878662, "kl": 0.4169921875, "learning_rate": 1.9296157727476448e-05, "loss": 0.0156, "num_tokens": 340625555.0, "reward": 0.6395089477300644, "reward_std": 0.10785190248861909, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.28471139818429947, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 667 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7142857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 991.3058471679688, "completions/mean_terminated_length": 913.0960998535156, "completions/min_length": 691.0, "completions/min_terminated_length": 691.0, "epoch": 0.19953700246434172, "grad_norm": 0.21836376190185547, "kl": 0.41796875, "learning_rate": 1.929213215650613e-05, "loss": 0.0171, "num_tokens": 341141356.0, "reward": 0.6651785969734192, "reward_std": 0.14524997863918543, "rewards/accuracy_reward/mean": 0.167410708963871, "rewards/accuracy_reward/std": 0.36645955592393875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678507566452, "rewards/tag_count_reward/std": 0.016628416255116463, "step": 668 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7388392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 988.8862152099609, "completions/mean_terminated_length": 889.4152526855469, "completions/min_length": 686.5, "completions/min_terminated_length": 686.5, "epoch": 0.1998357105518632, "grad_norm": 0.11885339766740799, "kl": 0.4658203125, "learning_rate": 1.9288095528719245e-05, "loss": 0.0213, "num_tokens": 341655401.0, "reward": 0.5647321492433548, "reward_std": 0.12497210130095482, "rewards/accuracy_reward/mean": 0.06919643096625805, "rewards/accuracy_reward/std": 0.23328668996691704, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027185317594558, "step": 669 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 997.5759582519531, "completions/mean_terminated_length": 883.4123229980469, "completions/min_length": 585.75, "completions/min_terminated_length": 585.75, "epoch": 0.20013441863938466, "grad_norm": 0.15649816393852234, "kl": 0.48876953125, "learning_rate": 1.9284047848919024e-05, "loss": 0.0196, "num_tokens": 342176971.0, "reward": 0.6093750298023224, "reward_std": 0.11495891958475113, "rewards/accuracy_reward/mean": 0.11383928591385484, "rewards/accuracy_reward/std": 0.28698471188545227, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 670 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.828125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 995.357177734375, "completions/mean_terminated_length": 869.0681610107422, "completions/min_length": 638.75, "completions/min_terminated_length": 638.75, "epoch": 0.20043312672690614, "grad_norm": 0.12449827045202255, "kl": 0.6357421875, "learning_rate": 1.9279989121921846e-05, "loss": 0.0272, "num_tokens": 342694539.0, "reward": 0.5664062649011612, "reward_std": 0.1286219246685505, "rewards/accuracy_reward/mean": 0.07589285587891936, "rewards/accuracy_reward/std": 0.25650588795542717, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983869820833, "step": 671 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 993.8103179931641, "completions/mean_terminated_length": 875.7855987548828, "completions/min_length": 670.5, "completions/min_terminated_length": 670.5, "epoch": 0.2007318348144276, "grad_norm": 0.5747271180152893, "kl": 0.45751953125, "learning_rate": 1.9275919352557242e-05, "loss": 0.0202, "num_tokens": 343212646.0, "reward": 0.6953125447034836, "reward_std": 0.1653020039666444, "rewards/accuracy_reward/mean": 0.2008928577415645, "rewards/accuracy_reward/std": 0.3527680039405823, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 672 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8013392857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 993.6986999511719, "completions/mean_terminated_length": 880.1831359863281, "completions/min_length": 585.75, "completions/min_terminated_length": 585.75, "epoch": 0.20103054290194908, "grad_norm": 0.44906511902809143, "kl": 0.728515625, "learning_rate": 1.9271838545667876e-05, "loss": 0.0354, "num_tokens": 343731135.0, "reward": 0.6484375298023224, "reward_std": 0.184870520606637, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.36356057971715927, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05717269517481327, "step": 673 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 1004.1562957763672, "completions/mean_terminated_length": 904.4472961425781, "completions/min_length": 734.0, "completions/min_terminated_length": 734.0, "epoch": 0.20132925098947055, "grad_norm": 0.2106211930513382, "kl": 1.24609375, "learning_rate": 1.9267746706109546e-05, "loss": 0.0547, "num_tokens": 344245445.0, "reward": 0.6272321790456772, "reward_std": 0.17981122620403767, "rewards/accuracy_reward/mean": 0.14955357182770967, "rewards/accuracy_reward/std": 0.2858026251196861, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4776785671710968, "rewards/tag_count_reward/std": 0.07070386968553066, "step": 674 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7857142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 981.8504943847656, "completions/mean_terminated_length": 829.9538726806641, "completions/min_length": 411.5, "completions/min_terminated_length": 411.5, "epoch": 0.20162795907699202, "grad_norm": 0.4635813534259796, "kl": 3.0859375, "learning_rate": 1.926364383875118e-05, "loss": 0.1249, "num_tokens": 344760722.0, "reward": 0.554129496216774, "reward_std": 0.17077183350920677, "rewards/accuracy_reward/mean": 0.10044642840512097, "rewards/accuracy_reward/std": 0.25709752552211285, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4536830335855484, "rewards/tag_count_reward/std": 0.09645429439842701, "step": 675 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8191964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 995.0960235595703, "completions/mean_terminated_length": 865.0103759765625, "completions/min_length": 611.25, "completions/min_terminated_length": 611.25, "epoch": 0.2019266671645135, "grad_norm": 0.5725810527801514, "kl": 4.74609375, "learning_rate": 1.9259529948474833e-05, "loss": 0.2035, "num_tokens": 345276093.0, "reward": 0.5530134290456772, "reward_std": 0.2372226044535637, "rewards/accuracy_reward/mean": 0.13839285960420966, "rewards/accuracy_reward/std": 0.2949652709066868, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4146205335855484, "rewards/tag_count_reward/std": 0.11865751631557941, "step": 676 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7075892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 956.1696929931641, "completions/mean_terminated_length": 804.2612609863281, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "epoch": 0.20222537525203496, "grad_norm": 0.3677419126033783, "kl": 2.53515625, "learning_rate": 1.9255405040175666e-05, "loss": 0.0959, "num_tokens": 345777417.0, "reward": 0.5669643133878708, "reward_std": 0.2126496247947216, "rewards/accuracy_reward/mean": 0.1406250037252903, "rewards/accuracy_reward/std": 0.3457693085074425, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4263392835855484, "rewards/tag_count_reward/std": 0.11346287652850151, "step": 677 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7410714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 976.5402221679688, "completions/mean_terminated_length": 840.829833984375, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "epoch": 0.20252408333955643, "grad_norm": 1.1344681978225708, "kl": 1.1591796875, "learning_rate": 1.9251269118761956e-05, "loss": 0.0558, "num_tokens": 346288763.0, "reward": 0.5541294887661934, "reward_std": 0.21319550275802612, "rewards/accuracy_reward/mean": 0.11160714365541935, "rewards/accuracy_reward/std": 0.2649957612156868, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4469866082072258, "rewards/tag_count_reward/std": 0.10114014707505703, "step": 678 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.703125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 968.1920013427734, "completions/mean_terminated_length": 841.0992126464844, "completions/min_length": 364.5, "completions/min_terminated_length": 364.5, "epoch": 0.20282279142707788, "grad_norm": 0.826041579246521, "kl": 1.513671875, "learning_rate": 1.9247122189155082e-05, "loss": 0.0718, "num_tokens": 346789537.0, "reward": 0.6060268133878708, "reward_std": 0.2175018284469843, "rewards/accuracy_reward/mean": 0.15848214458674192, "rewards/accuracy_reward/std": 0.35117682069540024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4475446343421936, "rewards/tag_count_reward/std": 0.10195181891322136, "step": 679 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 970.0826416015625, "completions/mean_terminated_length": 859.7086334228516, "completions/min_length": 431.25, "completions/min_terminated_length": 431.25, "epoch": 0.20312149951459935, "grad_norm": 1.0783355236053467, "kl": 2.47265625, "learning_rate": 1.924296425628953e-05, "loss": 0.0712, "num_tokens": 347294998.0, "reward": 0.5318080708384514, "reward_std": 0.1597901824861765, "rewards/accuracy_reward/mean": 0.060267859138548374, "rewards/accuracy_reward/std": 0.19628211855888367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4715401828289032, "rewards/tag_count_reward/std": 0.07828687038272619, "step": 680 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 933.0870971679688, "completions/mean_terminated_length": 823.0207824707031, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "epoch": 0.20342020760212082, "grad_norm": 0.5835758447647095, "kl": 1.80078125, "learning_rate": 1.9238795325112867e-05, "loss": 0.0639, "num_tokens": 347775965.0, "reward": 0.7098214626312256, "reward_std": 0.17218875885009766, "rewards/accuracy_reward/mean": 0.2384672649204731, "rewards/accuracy_reward/std": 0.410760335624218, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05837326589971781, "step": 681 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.75, "completions/mean_length": 950.1674499511719, "completions/mean_terminated_length": 843.9177856445312, "completions/min_length": 379.25, "completions/min_terminated_length": 379.25, "epoch": 0.2037189156896423, "grad_norm": 0.7901120781898499, "kl": 1.7890625, "learning_rate": 1.923461540058576e-05, "loss": 0.0419, "num_tokens": 348266760.0, "reward": 0.556361623108387, "reward_std": 0.1415780447423458, "rewards/accuracy_reward/mean": 0.08668155036866665, "rewards/accuracy_reward/std": 0.18919526413083076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4804687574505806, "rewards/tag_count_reward/std": 0.06593739800155163, "step": 682 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6584821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 968.8728179931641, "completions/mean_terminated_length": 865.7327575683594, "completions/min_length": 526.75, "completions/min_terminated_length": 526.75, "epoch": 0.20401762377716376, "grad_norm": 0.23359908163547516, "kl": 0.94140625, "learning_rate": 1.9230424487681944e-05, "loss": 0.0343, "num_tokens": 348778303.0, "reward": 0.6155134215950966, "reward_std": 0.11760617606341839, "rewards/accuracy_reward/mean": 0.12723214575089514, "rewards/accuracy_reward/std": 0.28350980393588543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04959508636966348, "step": 683 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.71875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 975.6451416015625, "completions/mean_terminated_length": 876.0792694091797, "completions/min_length": 650.25, "completions/min_terminated_length": 650.25, "epoch": 0.20431633186468523, "grad_norm": 0.2822510302066803, "kl": 0.470703125, "learning_rate": 1.9226222591388235e-05, "loss": 0.018, "num_tokens": 349289712.0, "reward": 0.567522332072258, "reward_std": 0.07878342270851135, "rewards/accuracy_reward/mean": 0.07589285867288709, "rewards/accuracy_reward/std": 0.2477288767695427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04293336346745491, "step": 684 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 983.0536041259766, "completions/mean_terminated_length": 890.8441314697266, "completions/min_length": 620.75, "completions/min_terminated_length": 620.75, "epoch": 0.2046150399522067, "grad_norm": 0.22672376036643982, "kl": 0.35791015625, "learning_rate": 1.922200971670452e-05, "loss": 0.0081, "num_tokens": 349807464.0, "reward": 0.7120536118745804, "reward_std": 0.1699868030846119, "rewards/accuracy_reward/mean": 0.22098214644938707, "rewards/accuracy_reward/std": 0.376902736723423, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044403897132724524, "step": 685 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7991071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 994.5491638183594, "completions/mean_terminated_length": 887.0942687988281, "completions/min_length": 666.25, "completions/min_terminated_length": 666.25, "epoch": 0.20491374803972817, "grad_norm": 0.21415449678897858, "kl": 0.302734375, "learning_rate": 1.921778586864375e-05, "loss": 0.011, "num_tokens": 350332302.0, "reward": 0.6160714626312256, "reward_std": 0.17071921564638615, "rewards/accuracy_reward/mean": 0.12499999813735485, "rewards/accuracy_reward/std": 0.32907411456108093, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.03981293365359306, "step": 686 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.5, "completions/mean_length": 990.4777069091797, "completions/mean_terminated_length": 885.8759002685547, "completions/min_length": 637.25, "completions/min_terminated_length": 637.25, "epoch": 0.20521245612724964, "grad_norm": 0.20450320839881897, "kl": 0.2890625, "learning_rate": 1.9213551052231925e-05, "loss": 0.0104, "num_tokens": 350846420.0, "reward": 0.663504496216774, "reward_std": 0.19152936339378357, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.37652380764484406, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.042347033973783255, "step": 687 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6473214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 964.0781707763672, "completions/mean_terminated_length": 860.333251953125, "completions/min_length": 573.5, "completions/min_terminated_length": 573.5, "epoch": 0.2055111642147711, "grad_norm": 0.21354049444198608, "kl": 0.300048828125, "learning_rate": 1.920930527250811e-05, "loss": 0.0023, "num_tokens": 351351623.0, "reward": 0.620535746216774, "reward_std": 0.07826446276158094, "rewards/accuracy_reward/mean": 0.12946428102441132, "rewards/accuracy_reward/std": 0.2962398882955313, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 688 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7544642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 995.8482666015625, "completions/mean_terminated_length": 906.0485382080078, "completions/min_length": 710.0, "completions/min_terminated_length": 710.0, "epoch": 0.20580987230229258, "grad_norm": 0.21468575298786163, "kl": 0.40283203125, "learning_rate": 1.9205048534524405e-05, "loss": 0.0192, "num_tokens": 351868003.0, "reward": 0.6300223618745804, "reward_std": 0.17167465575039387, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.3434209004044533, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.042347033973783255, "step": 689 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 978.4866485595703, "completions/mean_terminated_length": 869.8860931396484, "completions/min_length": 554.0, "completions/min_terminated_length": 554.0, "epoch": 0.20610858038981406, "grad_norm": 0.20743122696876526, "kl": 0.39892578125, "learning_rate": 1.920078084334595e-05, "loss": 0.0185, "num_tokens": 352380797.0, "reward": 0.6813616305589676, "reward_std": 0.15749619528651237, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.38511744141578674, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 690 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 989.7411193847656, "completions/mean_terminated_length": 890.0121459960938, "completions/min_length": 645.25, "completions/min_terminated_length": 645.25, "epoch": 0.20640728847733553, "grad_norm": 0.1867559552192688, "kl": 0.369140625, "learning_rate": 1.9196502204050925e-05, "loss": 0.0102, "num_tokens": 352892809.0, "reward": 0.5825893133878708, "reward_std": 0.09330978547222912, "rewards/accuracy_reward/mean": 0.08482142724096775, "rewards/accuracy_reward/std": 0.23854629695415497, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678656578064, "rewards/tag_count_reward/std": 0.023622779175639153, "step": 691 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 986.4531707763672, "completions/mean_terminated_length": 865.4737091064453, "completions/min_length": 606.75, "completions/min_terminated_length": 606.75, "epoch": 0.206705996564857, "grad_norm": 0.17271488904953003, "kl": 0.3779296875, "learning_rate": 1.9192212621730527e-05, "loss": 0.0158, "num_tokens": 353407252.0, "reward": 0.6774553954601288, "reward_std": 0.09851801674813032, "rewards/accuracy_reward/mean": 0.18080357019789517, "rewards/accuracy_reward/std": 0.3131284471601248, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 692 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8191964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 1000.6384429931641, "completions/mean_terminated_length": 896.3935699462891, "completions/min_length": 656.0, "completions/min_terminated_length": 656.0, "epoch": 0.20700470465237847, "grad_norm": 0.13374844193458557, "kl": 0.3408203125, "learning_rate": 1.9187912101488986e-05, "loss": 0.0169, "num_tokens": 353925154.0, "reward": 0.595982164144516, "reward_std": 0.12534409761428833, "rewards/accuracy_reward/mean": 0.10044642770662904, "rewards/accuracy_reward/std": 0.2836252562701702, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02827909868210554, "step": 693 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.828125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 1004.060302734375, "completions/mean_terminated_length": 909.1066131591797, "completions/min_length": 702.75, "completions/min_terminated_length": 702.75, "epoch": 0.20730341273989994, "grad_norm": 0.18482916057109833, "kl": 0.40087890625, "learning_rate": 1.9183600648443535e-05, "loss": 0.0214, "num_tokens": 354446877.0, "reward": 0.6110491454601288, "reward_std": 0.1298578903079033, "rewards/accuracy_reward/mean": 0.11830357019789517, "rewards/accuracy_reward/std": 0.2860332038253546, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 694 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7991071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 993.1674652099609, "completions/mean_terminated_length": 879.3895263671875, "completions/min_length": 666.75, "completions/min_terminated_length": 666.75, "epoch": 0.2076021208274214, "grad_norm": 0.1350753754377365, "kl": 0.33203125, "learning_rate": 1.917927826772443e-05, "loss": 0.0134, "num_tokens": 354967192.0, "reward": 0.6021205484867096, "reward_std": 0.06587101123295724, "rewards/accuracy_reward/mean": 0.1049107126891613, "rewards/accuracy_reward/std": 0.20404094457626343, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 695 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8459821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 1001.9464721679688, "completions/mean_terminated_length": 890.7579498291016, "completions/min_length": 694.0, "completions/min_terminated_length": 694.0, "epoch": 0.20790082891494288, "grad_norm": 0.1545642614364624, "kl": 0.56591796875, "learning_rate": 1.9174944964474914e-05, "loss": 0.0265, "num_tokens": 355488160.0, "reward": 0.595982164144516, "reward_std": 0.1646429286338389, "rewards/accuracy_reward/mean": 0.1026785708963871, "rewards/accuracy_reward/std": 0.2570536732673645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 696 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8236607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 1003.6004943847656, "completions/mean_terminated_length": 914.1965942382812, "completions/min_length": 721.75, "completions/min_terminated_length": 721.75, "epoch": 0.20819953700246435, "grad_norm": 0.13154685497283936, "kl": 0.455078125, "learning_rate": 1.917060074385124e-05, "loss": 0.0215, "num_tokens": 356019117.0, "reward": 0.6729910969734192, "reward_std": 0.1840583123266697, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.36901454254984856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 697 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 1000.1406707763672, "completions/mean_terminated_length": 914.349365234375, "completions/min_length": 777.0, "completions/min_terminated_length": 777.0, "epoch": 0.20849824508998582, "grad_norm": 0.1759842187166214, "kl": 0.6005859375, "learning_rate": 1.916624561102265e-05, "loss": 0.0213, "num_tokens": 356543324.0, "reward": 0.6819196790456772, "reward_std": 0.13510987535119057, "rewards/accuracy_reward/mean": 0.1919642835855484, "rewards/accuracy_reward/std": 0.3901069909334183, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04903263598680496, "step": 698 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8147321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 999.5156707763672, "completions/mean_terminated_length": 895.0100555419922, "completions/min_length": 710.25, "completions/min_terminated_length": 710.25, "epoch": 0.2087969531775073, "grad_norm": 0.1458783894777298, "kl": 0.5087890625, "learning_rate": 1.916187957117136e-05, "loss": 0.0226, "num_tokens": 357069843.0, "reward": 0.514508955180645, "reward_std": 0.07467832788825035, "rewards/accuracy_reward/mean": 0.022321428870782256, "rewards/accuracy_reward/std": 0.11766695789992809, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 699 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 1009.8370971679688, "completions/mean_terminated_length": 901.4408111572266, "completions/min_length": 670.0, "completions/min_terminated_length": 670.0, "epoch": 0.20909566126502874, "grad_norm": 0.18162642419338226, "kl": 0.61669921875, "learning_rate": 1.915750262949258e-05, "loss": 0.0268, "num_tokens": 357592730.0, "reward": 0.612723246216774, "reward_std": 0.1261256206780672, "rewards/accuracy_reward/mean": 0.12276785494759679, "rewards/accuracy_reward/std": 0.27761711180210114, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 700 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7946428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 983.950927734375, "completions/mean_terminated_length": 855.5733032226562, "completions/min_length": 592.0, "completions/min_terminated_length": 592.0, "epoch": 0.2093943693525502, "grad_norm": 0.18126459419727325, "kl": 0.759765625, "learning_rate": 1.9153114791194475e-05, "loss": 0.038, "num_tokens": 358102756.0, "reward": 0.5959821790456772, "reward_std": 0.12713690847158432, "rewards/accuracy_reward/mean": 0.10937500279396772, "rewards/accuracy_reward/std": 0.2957727462053299, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05471411347389221, "step": 701 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 992.0625305175781, "completions/mean_terminated_length": 866.9412078857422, "completions/min_length": 621.0, "completions/min_terminated_length": 621.0, "epoch": 0.20969307744007168, "grad_norm": 0.19538497924804688, "kl": 0.7353515625, "learning_rate": 1.9148716061498186e-05, "loss": 0.0339, "num_tokens": 358625024.0, "reward": 0.602120578289032, "reward_std": 0.11030163057148457, "rewards/accuracy_reward/mean": 0.10937499860301614, "rewards/accuracy_reward/std": 0.2908140830695629, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818479284644, "step": 702 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 1009.7745971679688, "completions/mean_terminated_length": 886.4562530517578, "completions/min_length": 651.0, "completions/min_terminated_length": 651.0, "epoch": 0.20999178552759315, "grad_norm": 0.1947515308856964, "kl": 0.8173828125, "learning_rate": 1.9144306445637822e-05, "loss": 0.0331, "num_tokens": 359151195.0, "reward": 0.5770089626312256, "reward_std": 0.1616463717073202, "rewards/accuracy_reward/mean": 0.08928571548312902, "rewards/accuracy_reward/std": 0.2761848084628582, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05642685014754534, "step": 703 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7790178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 993.0737152099609, "completions/mean_terminated_length": 883.6529846191406, "completions/min_length": 612.5, "completions/min_terminated_length": 612.5, "epoch": 0.21029049361511462, "grad_norm": 0.2315128892660141, "kl": 0.8388671875, "learning_rate": 1.913988594886043e-05, "loss": 0.0384, "num_tokens": 359672924.0, "reward": 0.6004464477300644, "reward_std": 0.12017786502838135, "rewards/accuracy_reward/mean": 0.11830357206054032, "rewards/accuracy_reward/std": 0.29585554264485836, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428656578064, "rewards/tag_count_reward/std": 0.06340559758245945, "step": 704 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7433035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 982.6205902099609, "completions/mean_terminated_length": 865.2744293212891, "completions/min_length": 514.75, "completions/min_terminated_length": 514.75, "epoch": 0.2105892017026361, "grad_norm": 2.566960334777832, "kl": 1.0205078125, "learning_rate": 1.913545457642601e-05, "loss": 0.0486, "num_tokens": 360190882.0, "reward": 0.500000037252903, "reward_std": 0.2046574503183365, "rewards/accuracy_reward/mean": 0.11607142630964518, "rewards/accuracy_reward/std": 0.31473907083272934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3839285746216774, "rewards/tag_count_reward/std": 0.12405591830611229, "step": 705 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7254464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 969.2589721679688, "completions/mean_terminated_length": 837.8986206054688, "completions/min_length": 536.25, "completions/min_terminated_length": 536.25, "epoch": 0.21088790979015756, "grad_norm": 0.3632712960243225, "kl": 0.7890625, "learning_rate": 1.9131012333607507e-05, "loss": 0.0408, "num_tokens": 360692662.0, "reward": 0.718191996216774, "reward_std": 0.18541576247662306, "rewards/accuracy_reward/mean": 0.23437500093132257, "rewards/accuracy_reward/std": 0.37981442362070084, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.060639471746981144, "step": 706 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 984.0692291259766, "completions/mean_terminated_length": 814.1037139892578, "completions/min_length": 465.0, "completions/min_terminated_length": 465.0, "epoch": 0.21118661787767903, "grad_norm": 4.706413269042969, "kl": 2.9921875, "learning_rate": 1.9126559225690796e-05, "loss": 0.1299, "num_tokens": 361206773.0, "reward": 0.4285714402794838, "reward_std": 0.17655162885785103, "rewards/accuracy_reward/mean": 0.04910714412108064, "rewards/accuracy_reward/std": 0.1756022684276104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.3794642835855484, "rewards/tag_count_reward/std": 0.1255180947482586, "step": 707 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 988.3326416015625, "completions/mean_terminated_length": 847.9342651367188, "completions/min_length": 607.25, "completions/min_terminated_length": 607.25, "epoch": 0.2114853259652005, "grad_norm": 0.4366910755634308, "kl": 0.77734375, "learning_rate": 1.9122095257974676e-05, "loss": 0.0354, "num_tokens": 361719210.0, "reward": 0.6149553954601288, "reward_std": 0.17720593325793743, "rewards/accuracy_reward/mean": 0.1428571422584355, "rewards/accuracy_reward/std": 0.3211631514132023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4720982164144516, "rewards/tag_count_reward/std": 0.07662395667284727, "step": 708 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 993.4308471679688, "completions/mean_terminated_length": 820.3474273681641, "completions/min_length": 375.25, "completions/min_terminated_length": 375.25, "epoch": 0.21178403405272198, "grad_norm": 0.6445937156677246, "kl": 0.779296875, "learning_rate": 1.911762043577089e-05, "loss": 0.03, "num_tokens": 362239851.0, "reward": 0.560825914144516, "reward_std": 0.19558723270893097, "rewards/accuracy_reward/mean": 0.09374999906867743, "rewards/accuracy_reward/std": 0.2358490228652954, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4670758917927742, "rewards/tag_count_reward/std": 0.08638879097998142, "step": 709 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.8058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.75, "completions/mean_length": 986.2210235595703, "completions/mean_terminated_length": 820.0626220703125, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 0.21208274214024345, "grad_norm": 0.3016664683818817, "kl": 0.9853515625, "learning_rate": 1.911313476440406e-05, "loss": 0.0255, "num_tokens": 362753966.0, "reward": 0.5641741305589676, "reward_std": 0.20209017768502235, "rewards/accuracy_reward/mean": 0.11383928544819355, "rewards/accuracy_reward/std": 0.314091544598341, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4503348246216774, "rewards/tag_count_reward/std": 0.0991473700851202, "step": 710 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 969.0246124267578, "completions/mean_terminated_length": 843.9755401611328, "completions/min_length": 539.75, "completions/min_terminated_length": 539.75, "epoch": 0.21238145022776492, "grad_norm": 0.3565990924835205, "kl": 1.3427734375, "learning_rate": 1.910863824921176e-05, "loss": 0.0578, "num_tokens": 363256889.0, "reward": 0.6685268133878708, "reward_std": 0.23409748077392578, "rewards/accuracy_reward/mean": 0.19866071455180645, "rewards/accuracy_reward/std": 0.3951898068189621, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4698660746216774, "rewards/tag_count_reward/std": 0.08139109797775745, "step": 711 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6517857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 954.8951416015625, "completions/mean_terminated_length": 841.6140747070312, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "epoch": 0.2126801583152864, "grad_norm": 0.3793347179889679, "kl": 2.037109375, "learning_rate": 1.9104130895544433e-05, "loss": 0.0957, "num_tokens": 363756826.0, "reward": 0.6981027126312256, "reward_std": 0.2405037321150303, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.40680375695228577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883992433548, "rewards/tag_count_reward/std": 0.07363102398812771, "step": 712 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7098214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 964.3125305175781, "completions/mean_terminated_length": 819.3236236572266, "completions/min_length": 444.5, "completions/min_terminated_length": 444.5, "epoch": 0.21297886640280786, "grad_norm": 0.6348779201507568, "kl": 3.0, "learning_rate": 1.9099612708765432e-05, "loss": 0.1277, "num_tokens": 364271910.0, "reward": 0.5915178805589676, "reward_std": 0.2032743487507105, "rewards/accuracy_reward/mean": 0.13020833348855376, "rewards/accuracy_reward/std": 0.31432870775461197, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4665178582072258, "rewards/tag_count_reward/std": 0.08328319527208805, "step": 713 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 969.2701416015625, "completions/mean_terminated_length": 836.0817718505859, "completions/min_length": 453.75, "completions/min_terminated_length": 453.75, "epoch": 0.21327757449032933, "grad_norm": 0.5222275853157043, "kl": 2.96875, "learning_rate": 1.9095083694251005e-05, "loss": 0.1207, "num_tokens": 364783583.0, "reward": 0.5926339626312256, "reward_std": 0.1861962527036667, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.31430598348379135, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.465401791036129, "rewards/tag_count_reward/std": 0.08501942455768585, "step": 714 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 966.5268249511719, "completions/mean_terminated_length": 840.6160736083984, "completions/min_length": 507.5, "completions/min_terminated_length": 507.5, "epoch": 0.2135762825778508, "grad_norm": 0.22389018535614014, "kl": 2.103515625, "learning_rate": 1.909054385739028e-05, "loss": 0.0947, "num_tokens": 365288987.0, "reward": 0.580915205180645, "reward_std": 0.15954991430044174, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.25508446991443634, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4715401753783226, "rewards/tag_count_reward/std": 0.07891392894089222, "step": 715 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 988.5893249511719, "completions/mean_terminated_length": 861.9590454101562, "completions/min_length": 598.25, "completions/min_terminated_length": 598.25, "epoch": 0.21387499066537227, "grad_norm": 0.40519070625305176, "kl": 1.865234375, "learning_rate": 1.9085993203585257e-05, "loss": 0.0819, "num_tokens": 365808195.0, "reward": 0.5820312798023224, "reward_std": 0.1938178353011608, "rewards/accuracy_reward/mean": 0.12723214272409678, "rewards/accuracy_reward/std": 0.3132109269499779, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4637276828289032, "rewards/tag_count_reward/std": 0.08890782669186592, "step": 716 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 962.6674499511719, "completions/mean_terminated_length": 863.0713653564453, "completions/min_length": 619.5, "completions/min_terminated_length": 619.5, "epoch": 0.21417369875289374, "grad_norm": 0.5050132274627686, "kl": 1.3994140625, "learning_rate": 1.9081431738250815e-05, "loss": 0.0661, "num_tokens": 366311598.0, "reward": 0.6668527126312256, "reward_std": 0.18764302507042885, "rewards/accuracy_reward/mean": 0.19196428591385484, "rewards/accuracy_reward/std": 0.3520175777375698, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883917927742, "rewards/tag_count_reward/std": 0.07372739166021347, "step": 717 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 951.3839874267578, "completions/mean_terminated_length": 844.1604766845703, "completions/min_length": 525.75, "completions/min_terminated_length": 525.75, "epoch": 0.21447240684041521, "grad_norm": 0.4250977039337158, "kl": 1.751953125, "learning_rate": 1.9076859466814692e-05, "loss": 0.085, "num_tokens": 366809978.0, "reward": 0.6383928805589676, "reward_std": 0.23955268040299416, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3627188205718994, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4709821417927742, "rewards/tag_count_reward/std": 0.08015618659555912, "step": 718 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 961.4576416015625, "completions/mean_terminated_length": 862.3913879394531, "completions/min_length": 546.25, "completions/min_terminated_length": 546.25, "epoch": 0.21477111492793668, "grad_norm": 0.38741299510002136, "kl": 2.525390625, "learning_rate": 1.9072276394717494e-05, "loss": 0.1155, "num_tokens": 367319239.0, "reward": 0.5719866156578064, "reward_std": 0.21802693232893944, "rewards/accuracy_reward/mean": 0.0982142873108387, "rewards/accuracy_reward/std": 0.2896936200559139, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4737723246216774, "rewards/tag_count_reward/std": 0.07529540173709393, "step": 719 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 940.904052734375, "completions/mean_terminated_length": 834.1309967041016, "completions/min_length": 444.25, "completions/min_terminated_length": 444.25, "epoch": 0.21506982301545816, "grad_norm": 0.28097981214523315, "kl": 2.646484375, "learning_rate": 1.9067682527412662e-05, "loss": 0.13, "num_tokens": 367810316.0, "reward": 0.5998883992433548, "reward_std": 0.2126959189772606, "rewards/accuracy_reward/mean": 0.1227678544819355, "rewards/accuracy_reward/std": 0.32144375890493393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4771205335855484, "rewards/tag_count_reward/std": 0.07329331338405609, "step": 720 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 951.9844207763672, "completions/mean_terminated_length": 837.6713104248047, "completions/min_length": 528.0, "completions/min_terminated_length": 528.0, "epoch": 0.21536853110297963, "grad_norm": 0.46031227707862854, "kl": 2.52734375, "learning_rate": 1.9063077870366504e-05, "loss": 0.1055, "num_tokens": 368305461.0, "reward": 0.5697544813156128, "reward_std": 0.1763580720871687, "rewards/accuracy_reward/mean": 0.09821428661234677, "rewards/accuracy_reward/std": 0.2644744049757719, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4715401828289032, "rewards/tag_count_reward/std": 0.07962732203304768, "step": 721 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6316964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 954.482177734375, "completions/mean_terminated_length": 837.3785552978516, "completions/min_length": 512.0, "completions/min_terminated_length": 512.0, "epoch": 0.21566723919050107, "grad_norm": 0.3259239196777344, "kl": 2.236328125, "learning_rate": 1.9058462429058143e-05, "loss": 0.1019, "num_tokens": 368809581.0, "reward": 0.6813616454601288, "reward_std": 0.25353650003671646, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.3980562388896942, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.476004458963871, "rewards/tag_count_reward/std": 0.07313574384897947, "step": 722 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 947.3817443847656, "completions/mean_terminated_length": 844.6653442382812, "completions/min_length": 501.75, "completions/min_terminated_length": 501.75, "epoch": 0.21596594727802254, "grad_norm": 0.237971231341362, "kl": 1.6328125, "learning_rate": 1.9053836208979554e-05, "loss": 0.0739, "num_tokens": 369298616.0, "reward": 0.5401785969734192, "reward_std": 0.12279621046036482, "rewards/accuracy_reward/mean": 0.06473214365541935, "rewards/accuracy_reward/std": 0.19122211635112762, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4754464328289032, "rewards/tag_count_reward/std": 0.07473543100059032, "step": 723 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7232142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 979.6629943847656, "completions/mean_terminated_length": 862.1515808105469, "completions/min_length": 573.5, "completions/min_terminated_length": 573.5, "epoch": 0.216264655365544, "grad_norm": 0.44376203417778015, "kl": 1.251953125, "learning_rate": 1.904919921563553e-05, "loss": 0.0498, "num_tokens": 369813361.0, "reward": 0.6378348469734192, "reward_std": 0.20655646175146103, "rewards/accuracy_reward/mean": 0.16406250279396772, "rewards/accuracy_reward/std": 0.3467092216014862, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4771205335855484, "rewards/tag_count_reward/std": 0.07202848792076111, "step": 724 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 948.5603179931641, "completions/mean_terminated_length": 848.3937530517578, "completions/min_length": 543.0, "completions/min_terminated_length": 543.0, "epoch": 0.21656336345306548, "grad_norm": 0.33753931522369385, "kl": 1.0283203125, "learning_rate": 1.9044551454543683e-05, "loss": 0.0502, "num_tokens": 370305852.0, "reward": 0.6367187798023224, "reward_std": 0.14609816297888756, "rewards/accuracy_reward/mean": 0.1495535671710968, "rewards/accuracy_reward/std": 0.3462056890130043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05343634728342295, "step": 725 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 945.6451416015625, "completions/mean_terminated_length": 845.8306732177734, "completions/min_length": 610.5, "completions/min_terminated_length": 610.5, "epoch": 0.21686207154058695, "grad_norm": 0.20126394927501678, "kl": 1.3291015625, "learning_rate": 1.9039892931234434e-05, "loss": 0.062, "num_tokens": 370802925.0, "reward": 0.5585937798023224, "reward_std": 0.10828488739207387, "rewards/accuracy_reward/mean": 0.07142857229337096, "rewards/accuracy_reward/std": 0.20885806530714035, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.053508032113313675, "step": 726 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 952.6674652099609, "completions/mean_terminated_length": 864.9611206054688, "completions/min_length": 591.25, "completions/min_terminated_length": 591.25, "epoch": 0.21716077962810842, "grad_norm": 0.31425556540489197, "kl": 1.697265625, "learning_rate": 1.903522365125102e-05, "loss": 0.0761, "num_tokens": 371300248.0, "reward": 0.569196455180645, "reward_std": 0.11439012736082077, "rewards/accuracy_reward/mean": 0.08258928684517741, "rewards/accuracy_reward/std": 0.22075842320919037, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 727 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 939.4620971679688, "completions/mean_terminated_length": 825.2185516357422, "completions/min_length": 429.5, "completions/min_terminated_length": 429.5, "epoch": 0.2174594877156299, "grad_norm": 0.3758692145347595, "kl": 1.7265625, "learning_rate": 1.903054362014947e-05, "loss": 0.0819, "num_tokens": 371796519.0, "reward": 0.588727705180645, "reward_std": 0.14352635154500604, "rewards/accuracy_reward/mean": 0.0982142835855484, "rewards/accuracy_reward/std": 0.19825376570224762, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 728 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5401785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 927.0960235595703, "completions/mean_terminated_length": 823.4991912841797, "completions/min_length": 505.75, "completions/min_terminated_length": 505.75, "epoch": 0.21775819580315137, "grad_norm": 0.4466324746608734, "kl": 1.322265625, "learning_rate": 1.902585284349861e-05, "loss": 0.0655, "num_tokens": 372277010.0, "reward": 0.6768973618745804, "reward_std": 0.14594364911317825, "rewards/accuracy_reward/mean": 0.18303571734577417, "rewards/accuracy_reward/std": 0.3250010423362255, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.04053540388122201, "step": 729 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 944.7433471679688, "completions/mean_terminated_length": 837.2158050537109, "completions/min_length": 521.25, "completions/min_terminated_length": 521.25, "epoch": 0.21805690389067284, "grad_norm": 0.3167928159236908, "kl": 1.1796875, "learning_rate": 1.902115132688004e-05, "loss": 0.0485, "num_tokens": 372770303.0, "reward": 0.6629464477300644, "reward_std": 0.17650193348526955, "rewards/accuracy_reward/mean": 0.17782738525420427, "rewards/accuracy_reward/std": 0.35790401697158813, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 730 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 948.4955749511719, "completions/mean_terminated_length": 838.2777252197266, "completions/min_length": 549.75, "completions/min_terminated_length": 549.75, "epoch": 0.2183556119781943, "grad_norm": 0.13357163965702057, "kl": 0.68017578125, "learning_rate": 1.901643907588816e-05, "loss": 0.0365, "num_tokens": 373266541.0, "reward": 0.5948661118745804, "reward_std": 0.11684337630867958, "rewards/accuracy_reward/mean": 0.10044643003493547, "rewards/accuracy_reward/std": 0.2869648188352585, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 731 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6919642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 967.7009429931641, "completions/mean_terminated_length": 860.9849700927734, "completions/min_length": 537.5, "completions/min_terminated_length": 537.5, "epoch": 0.21865432006571578, "grad_norm": 0.4865661859512329, "kl": 0.892578125, "learning_rate": 1.9011716096130132e-05, "loss": 0.049, "num_tokens": 373768119.0, "reward": 0.6847098618745804, "reward_std": 0.1769069116562605, "rewards/accuracy_reward/mean": 0.19642857229337096, "rewards/accuracy_reward/std": 0.3461848422884941, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05048930924385786, "step": 732 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6651785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 968.2031707763672, "completions/mean_terminated_length": 860.7180938720703, "completions/min_length": 520.25, "completions/min_terminated_length": 520.25, "epoch": 0.21895302815323725, "grad_norm": 0.27915525436401367, "kl": 0.7900390625, "learning_rate": 1.9006982393225878e-05, "loss": 0.0403, "num_tokens": 374274498.0, "reward": 0.6227678954601288, "reward_std": 0.1282023936510086, "rewards/accuracy_reward/mean": 0.13392857019789517, "rewards/accuracy_reward/std": 0.306873319670558, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04878514166921377, "step": 733 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5803571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 952.2879943847656, "completions/mean_terminated_length": 855.7017517089844, "completions/min_length": 608.25, "completions/min_terminated_length": 608.25, "epoch": 0.21925173624075872, "grad_norm": 0.22266051173210144, "kl": 0.57421875, "learning_rate": 1.9002237972808094e-05, "loss": 0.0251, "num_tokens": 374773619.0, "reward": 0.6579241305589676, "reward_std": 0.07895944872871041, "rewards/accuracy_reward/mean": 0.1692708358168602, "rewards/accuracy_reward/std": 0.35908081009984016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 734 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 988.4241485595703, "completions/mean_terminated_length": 867.1776123046875, "completions/min_length": 593.75, "completions/min_terminated_length": 593.75, "epoch": 0.2195504443282802, "grad_norm": 0.29725226759910583, "kl": 0.9580078125, "learning_rate": 1.8997482840522218e-05, "loss": 0.0453, "num_tokens": 375289617.0, "reward": 0.5915178805589676, "reward_std": 0.10386557737365365, "rewards/accuracy_reward/mean": 0.10044642863795161, "rewards/accuracy_reward/std": 0.23649465292692184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 735 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 944.2121124267578, "completions/mean_terminated_length": 829.2550659179688, "completions/min_length": 561.75, "completions/min_terminated_length": 561.75, "epoch": 0.21984915241580166, "grad_norm": 0.2173434942960739, "kl": 1.1494140625, "learning_rate": 1.8992717002026433e-05, "loss": 0.054, "num_tokens": 375782560.0, "reward": 0.5792410969734192, "reward_std": 0.1101080197840929, "rewards/accuracy_reward/mean": 0.08705357043072581, "rewards/accuracy_reward/std": 0.26829781010746956, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04052485013380647, "step": 736 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.25, "completions/mean_length": 945.8638763427734, "completions/mean_terminated_length": 824.9844207763672, "completions/min_length": 515.0, "completions/min_terminated_length": 515.0, "epoch": 0.22014786050332313, "grad_norm": 0.2712853252887726, "kl": 1.5087890625, "learning_rate": 1.8987940462991673e-05, "loss": 0.0738, "num_tokens": 376277763.0, "reward": 0.6015625149011612, "reward_std": 0.150643702596426, "rewards/accuracy_reward/mean": 0.1138392835855484, "rewards/accuracy_reward/std": 0.31898384541273117, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.051341270096600056, "step": 737 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 922.7054138183594, "completions/mean_terminated_length": 806.0116729736328, "completions/min_length": 450.0, "completions/min_terminated_length": 450.0, "epoch": 0.2204465685908446, "grad_norm": 0.790587842464447, "kl": 2.08984375, "learning_rate": 1.8983153229101592e-05, "loss": 0.0859, "num_tokens": 376767455.0, "reward": 0.5424107313156128, "reward_std": 0.09306294936686754, "rewards/accuracy_reward/mean": 0.05133928591385484, "rewards/accuracy_reward/std": 0.17836858704686165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 738 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 915.7098541259766, "completions/mean_terminated_length": 824.0519256591797, "completions/min_length": 510.0, "completions/min_terminated_length": 510.0, "epoch": 0.22074527667836608, "grad_norm": 0.6570075750350952, "kl": 2.03125, "learning_rate": 1.897835530605258e-05, "loss": 0.0908, "num_tokens": 377262861.0, "reward": 0.7170759290456772, "reward_std": 0.16010033711791039, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.40528130531311035, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 739 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 914.4308471679688, "completions/mean_terminated_length": 816.4101715087891, "completions/min_length": 496.0, "completions/min_terminated_length": 496.0, "epoch": 0.22104398476588755, "grad_norm": 0.30436211824417114, "kl": 1.23828125, "learning_rate": 1.8973546699553737e-05, "loss": 0.06, "num_tokens": 377740846.0, "reward": 0.5820312723517418, "reward_std": 0.1193480659276247, "rewards/accuracy_reward/mean": 0.09375000093132257, "rewards/accuracy_reward/std": 0.23424818366765976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.0502365012653172, "step": 740 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 920.9531707763672, "completions/mean_terminated_length": 824.5394744873047, "completions/min_length": 525.75, "completions/min_terminated_length": 525.75, "epoch": 0.22134269285340902, "grad_norm": 0.5490984320640564, "kl": 1.212890625, "learning_rate": 1.8968727415326885e-05, "loss": 0.0724, "num_tokens": 378222905.0, "reward": 0.6266741305589676, "reward_std": 0.15813351422548294, "rewards/accuracy_reward/mean": 0.1406250020954758, "rewards/accuracy_reward/std": 0.2880688887089491, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05486187245696783, "step": 741 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 877.6361999511719, "completions/mean_terminated_length": 763.0940399169922, "completions/min_length": 310.75, "completions/min_terminated_length": 310.75, "epoch": 0.2216414009409305, "grad_norm": 2.3510172367095947, "kl": 2.2373046875, "learning_rate": 1.8963897459106543e-05, "loss": 0.115, "num_tokens": 378687590.0, "reward": 0.585379496216774, "reward_std": 0.18043680489063263, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.2992083206772804, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4737723246216774, "rewards/tag_count_reward/std": 0.0717350197955966, "step": 742 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 922.9107513427734, "completions/mean_terminated_length": 816.0888366699219, "completions/min_length": 438.5, "completions/min_terminated_length": 438.5, "epoch": 0.22194010902845193, "grad_norm": 0.4925350248813629, "kl": 0.9052734375, "learning_rate": 1.8959056836639937e-05, "loss": 0.0465, "num_tokens": 379172254.0, "reward": 0.6183035969734192, "reward_std": 0.156558558344841, "rewards/accuracy_reward/mean": 0.1294642868451774, "rewards/accuracy_reward/std": 0.3177921324968338, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392984867096, "rewards/tag_count_reward/std": 0.051861658692359924, "step": 743 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 941.9464721679688, "completions/mean_terminated_length": 846.8408355712891, "completions/min_length": 539.25, "completions/min_terminated_length": 539.25, "epoch": 0.2222388171159734, "grad_norm": 0.4330897033214569, "kl": 1.080078125, "learning_rate": 1.895420555368697e-05, "loss": 0.0475, "num_tokens": 379664438.0, "reward": 0.6316964626312256, "reward_std": 0.21331458538770676, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.3465896435081959, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.0567212263122201, "step": 744 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 908.8973693847656, "completions/mean_terminated_length": 816.4479370117188, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "epoch": 0.22253752520349487, "grad_norm": 0.20619352161884308, "kl": 1.20703125, "learning_rate": 1.894934361602025e-05, "loss": 0.0522, "num_tokens": 380145816.0, "reward": 0.6132812798023224, "reward_std": 0.09637549426406622, "rewards/accuracy_reward/mean": 0.12500000232830644, "rewards/accuracy_reward/std": 0.23663589730858803, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052092005498707294, "step": 745 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 922.3147735595703, "completions/mean_terminated_length": 834.4483642578125, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "epoch": 0.22283623329101634, "grad_norm": 0.278168648481369, "kl": 1.6015625, "learning_rate": 1.8944471029425052e-05, "loss": 0.0702, "num_tokens": 380636645.0, "reward": 0.6277901977300644, "reward_std": 0.1273229829967022, "rewards/accuracy_reward/mean": 0.14062499906867743, "rewards/accuracy_reward/std": 0.3357275575399399, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05500977020710707, "step": 746 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 931.2768402099609, "completions/mean_terminated_length": 836.4541625976562, "completions/min_length": 471.5, "completions/min_terminated_length": 471.5, "epoch": 0.22313494137853782, "grad_norm": 0.6663123369216919, "kl": 1.845703125, "learning_rate": 1.8939587799699325e-05, "loss": 0.0733, "num_tokens": 381130993.0, "reward": 0.6741071790456772, "reward_std": 0.16407242277637124, "rewards/accuracy_reward/mean": 0.1830357201397419, "rewards/accuracy_reward/std": 0.3759728893637657, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194884091616, "step": 747 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 937.6629791259766, "completions/mean_terminated_length": 852.1285705566406, "completions/min_length": 446.0, "completions/min_terminated_length": 446.0, "epoch": 0.2234336494660593, "grad_norm": 0.39040014147758484, "kl": 2.6328125, "learning_rate": 1.893469393265367e-05, "loss": 0.1103, "num_tokens": 381622106.0, "reward": 0.5379464477300644, "reward_std": 0.1315806470811367, "rewards/accuracy_reward/mean": 0.05803571455180645, "rewards/accuracy_reward/std": 0.2197403870522976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4799107164144516, "rewards/tag_count_reward/std": 0.06735224463045597, "step": 748 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 925.7768402099609, "completions/mean_terminated_length": 836.0714416503906, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "epoch": 0.22373235755358076, "grad_norm": 0.34698235988616943, "kl": 1.619140625, "learning_rate": 1.892978943411137e-05, "loss": 0.0775, "num_tokens": 382101846.0, "reward": 0.6316964477300644, "reward_std": 0.16485249251127243, "rewards/accuracy_reward/mean": 0.14732142677530646, "rewards/accuracy_reward/std": 0.32099612802267075, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05888740811496973, "step": 749 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 930.3013916015625, "completions/mean_terminated_length": 831.9386596679688, "completions/min_length": 515.25, "completions/min_terminated_length": 515.25, "epoch": 0.22403106564110223, "grad_norm": 0.338662713766098, "kl": 1.4052734375, "learning_rate": 1.892487430990834e-05, "loss": 0.0579, "num_tokens": 382590893.0, "reward": 0.6456473469734192, "reward_std": 0.150582917034626, "rewards/accuracy_reward/mean": 0.16071428963914514, "rewards/accuracy_reward/std": 0.3322353698313236, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.058749482966959476, "step": 750 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.640625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 957.2857666015625, "completions/mean_terminated_length": 834.2085113525391, "completions/min_length": 604.25, "completions/min_terminated_length": 604.25, "epoch": 0.2243297737286237, "grad_norm": 0.41387107968330383, "kl": 1.0927734375, "learning_rate": 1.8919948565893144e-05, "loss": 0.0465, "num_tokens": 383086557.0, "reward": 0.665178582072258, "reward_std": 0.15210167691111565, "rewards/accuracy_reward/mean": 0.17857142724096775, "rewards/accuracy_reward/std": 0.36412907764315605, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05478769447654486, "step": 751 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 967.6875305175781, "completions/mean_terminated_length": 847.4007720947266, "completions/min_length": 531.5, "completions/min_terminated_length": 531.5, "epoch": 0.22462848181614517, "grad_norm": 0.29353970289230347, "kl": 2.03125, "learning_rate": 1.891501220792698e-05, "loss": 0.0886, "num_tokens": 383586561.0, "reward": 0.580357164144516, "reward_std": 0.16443004831671715, "rewards/accuracy_reward/mean": 0.10342262266203761, "rewards/accuracy_reward/std": 0.22933275252580643, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4776785597205162, "rewards/tag_count_reward/std": 0.0702384989708662, "step": 752 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7566964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 995.0067443847656, "completions/mean_terminated_length": 909.9714965820312, "completions/min_length": 704.5, "completions/min_terminated_length": 704.5, "epoch": 0.22492718990366664, "grad_norm": 0.3600198030471802, "kl": 2.306640625, "learning_rate": 1.891006524188368e-05, "loss": 0.0996, "num_tokens": 384109396.0, "reward": 0.6110491454601288, "reward_std": 0.19272762909531593, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.27798473089933395, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4726562425494194, "rewards/tag_count_reward/std": 0.07760501652956009, "step": 753 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6852678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 977.4397735595703, "completions/mean_terminated_length": 879.9359436035156, "completions/min_length": 607.25, "completions/min_terminated_length": 607.25, "epoch": 0.2252258979911881, "grad_norm": 0.3173944652080536, "kl": 2.5, "learning_rate": 1.89051076736497e-05, "loss": 0.1081, "num_tokens": 384618153.0, "reward": 0.6467634439468384, "reward_std": 0.23743607476353645, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.3662288784980774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883917927742, "rewards/tag_count_reward/std": 0.07472348213195801, "step": 754 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7321428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 979.3995971679688, "completions/mean_terminated_length": 884.7303009033203, "completions/min_length": 601.75, "completions/min_terminated_length": 601.75, "epoch": 0.22552460607870958, "grad_norm": 0.3747740685939789, "kl": 3.05859375, "learning_rate": 1.89001395091241e-05, "loss": 0.1378, "num_tokens": 385125756.0, "reward": 0.623325914144516, "reward_std": 0.24896317347884178, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.34910179674625397, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4693080335855484, "rewards/tag_count_reward/std": 0.08214361779391766, "step": 755 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 965.5223541259766, "completions/mean_terminated_length": 854.6930541992188, "completions/min_length": 597.0, "completions/min_terminated_length": 597.0, "epoch": 0.22582331416623105, "grad_norm": 0.4082246422767639, "kl": 3.08203125, "learning_rate": 1.8895160754218562e-05, "loss": 0.1274, "num_tokens": 385631942.0, "reward": 0.6512277126312256, "reward_std": 0.16751939617097378, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.37967392057180405, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883992433548, "rewards/tag_count_reward/std": 0.0736120967194438, "step": 756 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7321428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 971.8839721679688, "completions/mean_terminated_length": 838.0871429443359, "completions/min_length": 533.25, "completions/min_terminated_length": 533.25, "epoch": 0.22612202225375252, "grad_norm": 0.5456766486167908, "kl": 2.490234375, "learning_rate": 1.8890171414857366e-05, "loss": 0.1186, "num_tokens": 386143106.0, "reward": 0.6517857387661934, "reward_std": 0.1794949285686016, "rewards/accuracy_reward/mean": 0.1808035676367581, "rewards/accuracy_reward/std": 0.3402502126991749, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4709821492433548, "rewards/tag_count_reward/std": 0.07719200849533081, "step": 757 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7165178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 978.450927734375, "completions/mean_terminated_length": 865.6671905517578, "completions/min_length": 553.0, "completions/min_terminated_length": 553.0, "epoch": 0.226420730341274, "grad_norm": 0.29631057381629944, "kl": 2.4609375, "learning_rate": 1.8885171496977382e-05, "loss": 0.1021, "num_tokens": 386655820.0, "reward": 0.5993303954601288, "reward_std": 0.1968697514384985, "rewards/accuracy_reward/mean": 0.12499999767169356, "rewards/accuracy_reward/std": 0.3006262257695198, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4743303656578064, "rewards/tag_count_reward/std": 0.0754140829667449, "step": 758 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7165178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 965.5335235595703, "completions/mean_terminated_length": 845.0530700683594, "completions/min_length": 561.75, "completions/min_terminated_length": 561.75, "epoch": 0.22671943842879547, "grad_norm": 0.5365000367164612, "kl": 2.33203125, "learning_rate": 1.8880161006528075e-05, "loss": 0.1053, "num_tokens": 387161019.0, "reward": 0.5457589402794838, "reward_std": 0.16090983524918556, "rewards/accuracy_reward/mean": 0.07142857136204839, "rewards/accuracy_reward/std": 0.19770153984427452, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4743303582072258, "rewards/tag_count_reward/std": 0.07593118399381638, "step": 759 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 954.3549499511719, "completions/mean_terminated_length": 851.9760894775391, "completions/min_length": 549.75, "completions/min_terminated_length": 549.75, "epoch": 0.22701814651631694, "grad_norm": 0.3093374967575073, "kl": 2.19921875, "learning_rate": 1.887513994947148e-05, "loss": 0.0976, "num_tokens": 387664330.0, "reward": 0.5446428805589676, "reward_std": 0.15320205688476562, "rewards/accuracy_reward/mean": 0.0625, "rewards/accuracy_reward/std": 0.23544743284583092, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06410299614071846, "step": 760 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 931.8460235595703, "completions/mean_terminated_length": 831.3900756835938, "completions/min_length": 475.25, "completions/min_terminated_length": 475.25, "epoch": 0.2273168546038384, "grad_norm": 0.5597169399261475, "kl": 1.90625, "learning_rate": 1.887010833178222e-05, "loss": 0.0832, "num_tokens": 388157429.0, "reward": 0.6679687649011612, "reward_std": 0.1694355048239231, "rewards/accuracy_reward/mean": 0.17857143213041127, "rewards/accuracy_reward/std": 0.3329729624092579, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.048003614880144596, "step": 761 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 902.6562957763672, "completions/mean_terminated_length": 817.2791900634766, "completions/min_length": 517.75, "completions/min_terminated_length": 517.75, "epoch": 0.22761556269135988, "grad_norm": 0.36283889412879944, "kl": 1.904296875, "learning_rate": 1.8865066159447468e-05, "loss": 0.0892, "num_tokens": 388634235.0, "reward": 0.5675223544239998, "reward_std": 0.10707707889378071, "rewards/accuracy_reward/mean": 0.07812500046566129, "rewards/accuracy_reward/std": 0.19588801264762878, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 762 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 889.9754791259766, "completions/mean_terminated_length": 781.7105407714844, "completions/min_length": 435.25, "completions/min_terminated_length": 435.25, "epoch": 0.22791427077888135, "grad_norm": 0.23991380631923676, "kl": 1.162109375, "learning_rate": 1.8860013438466966e-05, "loss": 0.0593, "num_tokens": 389102848.0, "reward": 0.7449776977300644, "reward_std": 0.14233321696519852, "rewards/accuracy_reward/mean": 0.2522321380674839, "rewards/accuracy_reward/std": 0.4320768341422081, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04000696213915944, "step": 763 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 889.5067291259766, "completions/mean_terminated_length": 799.2042083740234, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "epoch": 0.22821297886640282, "grad_norm": 0.25018641352653503, "kl": 1.142578125, "learning_rate": 1.8854950174853003e-05, "loss": 0.0579, "num_tokens": 389571123.0, "reward": 0.6450893133878708, "reward_std": 0.1713125742971897, "rewards/accuracy_reward/mean": 0.15252975886687636, "rewards/accuracy_reward/std": 0.3177761510014534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035597205162, "rewards/tag_count_reward/std": 0.03907900024205446, "step": 764 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 918.2210235595703, "completions/mean_terminated_length": 816.5526275634766, "completions/min_length": 505.5, "completions/min_terminated_length": 505.5, "epoch": 0.22851168695392426, "grad_norm": 0.2761167287826538, "kl": 0.845703125, "learning_rate": 1.884987637463042e-05, "loss": 0.0412, "num_tokens": 390059926.0, "reward": 0.5898437649011612, "reward_std": 0.08786613959819078, "rewards/accuracy_reward/mean": 0.09374999767169356, "rewards/accuracy_reward/std": 0.26958027854561806, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.026178478728979826, "step": 765 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 894.7098693847656, "completions/mean_terminated_length": 810.9898223876953, "completions/min_length": 473.25, "completions/min_terminated_length": 473.25, "epoch": 0.22881039504144574, "grad_norm": 0.21043440699577332, "kl": 0.6572265625, "learning_rate": 1.8844792043836592e-05, "loss": 0.028, "num_tokens": 390531220.0, "reward": 0.7243303954601288, "reward_std": 0.1867387741804123, "rewards/accuracy_reward/mean": 0.2276785708963871, "rewards/accuracy_reward/std": 0.41983721405267715, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.027853476349264383, "step": 766 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 871.7723541259766, "completions/mean_terminated_length": 770.9035339355469, "completions/min_length": 464.5, "completions/min_terminated_length": 464.5, "epoch": 0.2291091031289672, "grad_norm": 0.3014867603778839, "kl": 0.5634765625, "learning_rate": 1.8839697188521416e-05, "loss": 0.0234, "num_tokens": 390987710.0, "reward": 0.713169664144516, "reward_std": 0.15782358311116695, "rewards/accuracy_reward/mean": 0.2187500004656613, "rewards/accuracy_reward/std": 0.35301613435149193, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.034913196228444576, "step": 767 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 930.6741485595703, "completions/mean_terminated_length": 850.1403961181641, "completions/min_length": 511.0, "completions/min_terminated_length": 511.0, "epoch": 0.22940781121648868, "grad_norm": 0.4220792353153229, "kl": 0.623046875, "learning_rate": 1.883459181474733e-05, "loss": 0.0312, "num_tokens": 391478940.0, "reward": 0.5859375223517418, "reward_std": 0.0997647549957037, "rewards/accuracy_reward/mean": 0.09151785727590322, "rewards/accuracy_reward/std": 0.23008278757333755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.030178462620824575, "step": 768 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 954.2299499511719, "completions/mean_terminated_length": 849.5719146728516, "completions/min_length": 560.0, "completions/min_terminated_length": 560.0, "epoch": 0.22970651930401015, "grad_norm": 0.5201273560523987, "kl": 0.8818359375, "learning_rate": 1.8829475928589272e-05, "loss": 0.0484, "num_tokens": 391979747.0, "reward": 0.6690848618745804, "reward_std": 0.216446653008461, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.37802574783563614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05167329590767622, "step": 769 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 949.8750305175781, "completions/mean_terminated_length": 857.1660614013672, "completions/min_length": 574.5, "completions/min_terminated_length": 574.5, "epoch": 0.23000522739153162, "grad_norm": 0.42384013533592224, "kl": 1.2275390625, "learning_rate": 1.8824349536134694e-05, "loss": 0.0572, "num_tokens": 392479451.0, "reward": 0.6004464477300644, "reward_std": 0.13562762923538685, "rewards/accuracy_reward/mean": 0.11755952425301075, "rewards/accuracy_reward/std": 0.3224317580461502, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.0534196263179183, "step": 770 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 942.5424499511719, "completions/mean_terminated_length": 827.9914245605469, "completions/min_length": 507.5, "completions/min_terminated_length": 507.5, "epoch": 0.2303039354790531, "grad_norm": 0.38376447558403015, "kl": 1.845703125, "learning_rate": 1.881921264348355e-05, "loss": 0.0882, "num_tokens": 392973022.0, "reward": 0.6668527126312256, "reward_std": 0.16956760454922915, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.3834410309791565, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06091840658336878, "step": 771 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6450892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 949.3370819091797, "completions/mean_terminated_length": 815.1538848876953, "completions/min_length": 419.25, "completions/min_terminated_length": 419.25, "epoch": 0.23060264356657456, "grad_norm": 1.12949800491333, "kl": 3.9609375, "learning_rate": 1.8814065256748294e-05, "loss": 0.1704, "num_tokens": 393473893.0, "reward": 0.5859375298023224, "reward_std": 0.15865741483867168, "rewards/accuracy_reward/mean": 0.10714285913854837, "rewards/accuracy_reward/std": 0.2943919748067856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.06875326298177242, "step": 772 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 920.1339721679688, "completions/mean_terminated_length": 795.4506530761719, "completions/min_length": 408.75, "completions/min_terminated_length": 408.75, "epoch": 0.23090135165409603, "grad_norm": 1.2557188272476196, "kl": 4.56640625, "learning_rate": 1.880890738205386e-05, "loss": 0.1897, "num_tokens": 393965281.0, "reward": 0.638950914144516, "reward_std": 0.2607831209897995, "rewards/accuracy_reward/mean": 0.1696428544819355, "rewards/accuracy_reward/std": 0.36954037100076675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4693080335855484, "rewards/tag_count_reward/std": 0.08153771050274372, "step": 773 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 935.6674652099609, "completions/mean_terminated_length": 805.7378997802734, "completions/min_length": 351.5, "completions/min_terminated_length": 351.5, "epoch": 0.2312000597416175, "grad_norm": 0.5303698182106018, "kl": 1.751953125, "learning_rate": 1.8803739025537655e-05, "loss": 0.0741, "num_tokens": 394456572.0, "reward": 0.5390625298023224, "reward_std": 0.11276136199012399, "rewards/accuracy_reward/mean": 0.05803571408614516, "rewards/accuracy_reward/std": 0.19186366349458694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06327600497752428, "step": 774 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 900.904052734375, "completions/mean_terminated_length": 778.4661865234375, "completions/min_length": 329.75, "completions/min_terminated_length": 329.75, "epoch": 0.23149876782913897, "grad_norm": 0.38664549589157104, "kl": 0.83984375, "learning_rate": 1.8798560193349575e-05, "loss": 0.0197, "num_tokens": 394927777.0, "reward": 0.7237723618745804, "reward_std": 0.15909090638160706, "rewards/accuracy_reward/mean": 0.23437499813735485, "rewards/accuracy_reward/std": 0.37924617528915405, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05165327154099941, "step": 775 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 939.6897583007812, "completions/mean_terminated_length": 832.8565521240234, "completions/min_length": 408.75, "completions/min_terminated_length": 408.75, "epoch": 0.23179747591666044, "grad_norm": 0.4774095416069031, "kl": 0.7900390625, "learning_rate": 1.8793370891651973e-05, "loss": 0.0293, "num_tokens": 395422790.0, "reward": 0.6344866454601288, "reward_std": 0.20091502368450165, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.3404533378779888, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 776 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 906.8304138183594, "completions/mean_terminated_length": 800.3036041259766, "completions/min_length": 431.5, "completions/min_terminated_length": 431.5, "epoch": 0.23209618400418192, "grad_norm": 0.5356490015983582, "kl": 1.1005859375, "learning_rate": 1.8788171126619653e-05, "loss": 0.0445, "num_tokens": 395909962.0, "reward": 0.599888414144516, "reward_std": 0.13323591835796833, "rewards/accuracy_reward/mean": 0.11160713993012905, "rewards/accuracy_reward/std": 0.3146684020757675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918200105429, "step": 777 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 891.9866638183594, "completions/mean_terminated_length": 790.5148773193359, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.2323948920917034, "grad_norm": 0.42144134640693665, "kl": 1.013671875, "learning_rate": 1.8782960904439887e-05, "loss": 0.0464, "num_tokens": 396376644.0, "reward": 0.646763414144516, "reward_std": 0.1626942977309227, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.2966117560863495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03516172803938389, "step": 778 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 915.2076263427734, "completions/mean_terminated_length": 799.0364990234375, "completions/min_length": 315.75, "completions/min_terminated_length": 315.75, "epoch": 0.23269360017922486, "grad_norm": 0.29438844323158264, "kl": 1.2861328125, "learning_rate": 1.877774023131237e-05, "loss": 0.0638, "num_tokens": 396860689.0, "reward": 0.6188616156578064, "reward_std": 0.1598268747329712, "rewards/accuracy_reward/mean": 0.12723214109428227, "rewards/accuracy_reward/std": 0.2995477579534054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 779 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 937.3013763427734, "completions/mean_terminated_length": 841.6874237060547, "completions/min_length": 499.5, "completions/min_terminated_length": 499.5, "epoch": 0.23299230826674633, "grad_norm": 0.3672548532485962, "kl": 2.14453125, "learning_rate": 1.8772509113449243e-05, "loss": 0.0925, "num_tokens": 397350008.0, "reward": 0.6043526977300644, "reward_std": 0.17746189050376415, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.3239203244447708, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05749546363949776, "step": 780 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 908.1987152099609, "completions/mean_terminated_length": 786.2890014648438, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "epoch": 0.2332910163542678, "grad_norm": 0.7436572909355164, "kl": 2.1953125, "learning_rate": 1.876726755707508e-05, "loss": 0.0932, "num_tokens": 397832497.0, "reward": 0.6065848618745804, "reward_std": 0.1192165375687182, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.26292797178030014, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983776688576, "step": 781 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6495535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 949.5803985595703, "completions/mean_terminated_length": 811.7656707763672, "completions/min_length": 515.75, "completions/min_terminated_length": 515.75, "epoch": 0.23358972444178927, "grad_norm": 0.5699330568313599, "kl": 2.021484375, "learning_rate": 1.8762015568426862e-05, "loss": 0.0788, "num_tokens": 398329205.0, "reward": 0.6289062798023224, "reward_std": 0.14503166265785694, "rewards/accuracy_reward/mean": 0.1450892835855484, "rewards/accuracy_reward/std": 0.29436157643795013, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.0607560845091939, "step": 782 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6540178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 956.3527374267578, "completions/mean_terminated_length": 828.5774078369141, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.23388843252931074, "grad_norm": 0.22292302548885345, "kl": 1.3642578125, "learning_rate": 1.8756753153753998e-05, "loss": 0.0464, "num_tokens": 398827155.0, "reward": 0.6294643133878708, "reward_std": 0.20181473903357983, "rewards/accuracy_reward/mean": 0.14285714412108064, "rewards/accuracy_reward/std": 0.3166483901441097, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 783 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6584821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 962.8638763427734, "completions/mean_terminated_length": 864.6197509765625, "completions/min_length": 614.25, "completions/min_terminated_length": 614.25, "epoch": 0.2341871406168322, "grad_norm": 0.3977932929992676, "kl": 1.544921875, "learning_rate": 1.8751480319318296e-05, "loss": 0.0709, "num_tokens": 399331046.0, "reward": 0.6250000298023224, "reward_std": 0.18950475379824638, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.3449920415878296, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428507566452, "rewards/tag_count_reward/std": 0.06402330100536346, "step": 784 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 975.2745971679688, "completions/mean_terminated_length": 871.4198608398438, "completions/min_length": 571.75, "completions/min_terminated_length": 571.75, "epoch": 0.23448584870435368, "grad_norm": 0.2522177994251251, "kl": 1.556640625, "learning_rate": 1.874619707139396e-05, "loss": 0.0642, "num_tokens": 399842481.0, "reward": 0.5898437798023224, "reward_std": 0.16631951741874218, "rewards/accuracy_reward/mean": 0.10937499953433871, "rewards/accuracy_reward/std": 0.24151474237442017, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4804687425494194, "rewards/tag_count_reward/std": 0.06723255384713411, "step": 785 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 987.6875457763672, "completions/mean_terminated_length": 875.6708526611328, "completions/min_length": 614.25, "completions/min_terminated_length": 614.25, "epoch": 0.23478455679187513, "grad_norm": 0.6019185185432434, "kl": 1.84375, "learning_rate": 1.874090341626759e-05, "loss": 0.0793, "num_tokens": 400370581.0, "reward": 0.587053582072258, "reward_std": 0.17433506064116955, "rewards/accuracy_reward/mean": 0.11830357322469354, "rewards/accuracy_reward/std": 0.2980150915682316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.46875, "rewards/tag_count_reward/std": 0.08225569315254688, "step": 786 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6919642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 969.1942443847656, "completions/mean_terminated_length": 860.988037109375, "completions/min_length": 529.0, "completions/min_terminated_length": 529.0, "epoch": 0.2350832648793966, "grad_norm": 0.38305720686912537, "kl": 2.55078125, "learning_rate": 1.873559936023817e-05, "loss": 0.1132, "num_tokens": 400877836.0, "reward": 0.5809152200818062, "reward_std": 0.18402566015720367, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.26292797178030014, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4648437425494194, "rewards/tag_count_reward/std": 0.0863427184522152, "step": 787 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 947.0781555175781, "completions/mean_terminated_length": 821.0869140625, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "epoch": 0.23538197296691807, "grad_norm": 0.7927695512771606, "kl": 3.94140625, "learning_rate": 1.8730284909617053e-05, "loss": 0.1735, "num_tokens": 401377199.0, "reward": 0.570312537252903, "reward_std": 0.16587468795478344, "rewards/accuracy_reward/mean": 0.11086309608072042, "rewards/accuracy_reward/std": 0.25127797573804855, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4609375, "rewards/tag_count_reward/std": 0.09033600427210331, "step": 788 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7254464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 974.5580596923828, "completions/mean_terminated_length": 843.9206848144531, "completions/min_length": 478.25, "completions/min_terminated_length": 478.25, "epoch": 0.23568068105443954, "grad_norm": 0.468001127243042, "kl": 4.0234375, "learning_rate": 1.8724960070727974e-05, "loss": 0.1704, "num_tokens": 401885033.0, "reward": 0.5044642984867096, "reward_std": 0.18698393553495407, "rewards/accuracy_reward/mean": 0.05803571501746774, "rewards/accuracy_reward/std": 0.22948775067925453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4464285746216774, "rewards/tag_count_reward/std": 0.10295183770358562, "step": 789 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 948.1250457763672, "completions/mean_terminated_length": 836.4688110351562, "completions/min_length": 437.25, "completions/min_terminated_length": 437.25, "epoch": 0.235979389141961, "grad_norm": 0.2543919086456299, "kl": 3.0, "learning_rate": 1.8719624849907013e-05, "loss": 0.1373, "num_tokens": 402386097.0, "reward": 0.6099330633878708, "reward_std": 0.23459670692682266, "rewards/accuracy_reward/mean": 0.1450892868451774, "rewards/accuracy_reward/std": 0.3275037333369255, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.46484375, "rewards/tag_count_reward/std": 0.08665353804826736, "step": 790 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 944.2567443847656, "completions/mean_terminated_length": 820.47509765625, "completions/min_length": 503.75, "completions/min_terminated_length": 503.75, "epoch": 0.23627809722948248, "grad_norm": 0.5582000017166138, "kl": 2.859375, "learning_rate": 1.8714279253502616e-05, "loss": 0.1411, "num_tokens": 402880404.0, "reward": 0.5574776977300644, "reward_std": 0.18586749769747257, "rewards/accuracy_reward/mean": 0.0982142835855484, "rewards/accuracy_reward/std": 0.2531045451760292, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4592633917927742, "rewards/tag_count_reward/std": 0.09151420556008816, "step": 791 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 950.0223541259766, "completions/mean_terminated_length": 826.5106201171875, "completions/min_length": 449.0, "completions/min_terminated_length": 449.0, "epoch": 0.23657680531700395, "grad_norm": 0.40954282879829407, "kl": 2.935546875, "learning_rate": 1.8708923287875568e-05, "loss": 0.1321, "num_tokens": 403372446.0, "reward": 0.6445312649011612, "reward_std": 0.24508266896009445, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.38250135630369186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.461495541036129, "rewards/tag_count_reward/std": 0.09003669768571854, "step": 792 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 952.8527374267578, "completions/mean_terminated_length": 839.7287750244141, "completions/min_length": 438.5, "completions/min_terminated_length": 438.5, "epoch": 0.23687551340452542, "grad_norm": 0.32381442189216614, "kl": 3.4140625, "learning_rate": 1.8703556959398998e-05, "loss": 0.1447, "num_tokens": 403869196.0, "reward": 0.5786830708384514, "reward_std": 0.17640767246484756, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.26765289157629013, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4670758992433548, "rewards/tag_count_reward/std": 0.08410260267555714, "step": 793 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 856.7612152099609, "completions/mean_terminated_length": 766.0725555419922, "completions/min_length": 222.25, "completions/min_terminated_length": 222.25, "epoch": 0.2371742214920469, "grad_norm": 0.9141747951507568, "kl": 3.6953125, "learning_rate": 1.8698180274458362e-05, "loss": 0.1622, "num_tokens": 404318545.0, "reward": 0.5909598469734192, "reward_std": 0.2013346292078495, "rewards/accuracy_reward/mean": 0.11607143213041127, "rewards/accuracy_reward/std": 0.2755274251103401, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883917927742, "rewards/tag_count_reward/std": 0.0750711802393198, "step": 794 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 890.3839721679688, "completions/mean_terminated_length": 775.1673278808594, "completions/min_length": 198.0, "completions/min_terminated_length": 198.0, "epoch": 0.23747292957956836, "grad_norm": 0.7631299495697021, "kl": 2.96484375, "learning_rate": 1.869279323945144e-05, "loss": 0.1141, "num_tokens": 404792221.0, "reward": 0.6462053805589676, "reward_std": 0.2199167013168335, "rewards/accuracy_reward/mean": 0.1696428544819355, "rewards/accuracy_reward/std": 0.37386611849069595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4765625, "rewards/tag_count_reward/std": 0.07179595343768597, "step": 795 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 848.9286193847656, "completions/mean_terminated_length": 755.0001983642578, "completions/min_length": 179.5, "completions/min_terminated_length": 179.5, "epoch": 0.23777163766708984, "grad_norm": 0.3836016058921814, "kl": 1.384765625, "learning_rate": 1.8687395860788325e-05, "loss": 0.0812, "num_tokens": 405244925.0, "reward": 0.6573661118745804, "reward_std": 0.18097873777151108, "rewards/accuracy_reward/mean": 0.1696428619325161, "rewards/accuracy_reward/std": 0.36616289615631104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053695037961006165, "step": 796 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 849.0402069091797, "completions/mean_terminated_length": 743.8262786865234, "completions/min_length": 400.75, "completions/min_terminated_length": 400.75, "epoch": 0.2380703457546113, "grad_norm": 0.41278332471847534, "kl": 0.796875, "learning_rate": 1.8681988144891425e-05, "loss": 0.0382, "num_tokens": 405693663.0, "reward": 0.5691964477300644, "reward_std": 0.10166086070239544, "rewards/accuracy_reward/mean": 0.07812499930150807, "rewards/accuracy_reward/std": 0.20207696966826916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04335387283936143, "step": 797 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 909.2812805175781, "completions/mean_terminated_length": 819.8307495117188, "completions/min_length": 518.75, "completions/min_terminated_length": 518.75, "epoch": 0.23836905384213278, "grad_norm": 0.4626450538635254, "kl": 0.54052734375, "learning_rate": 1.8676570098195443e-05, "loss": 0.0358, "num_tokens": 406176061.0, "reward": 0.6238839626312256, "reward_std": 0.16590415174141526, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.28411048650741577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 798 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 888.2611999511719, "completions/mean_terminated_length": 798.2276916503906, "completions/min_length": 352.5, "completions/min_terminated_length": 352.5, "epoch": 0.23866776192965425, "grad_norm": 0.16216282546520233, "kl": 0.243896484375, "learning_rate": 1.8671141727147374e-05, "loss": 0.0001, "num_tokens": 406638818.0, "reward": 0.715401828289032, "reward_std": 0.15279777348041534, "rewards/accuracy_reward/mean": 0.231398805975914, "rewards/accuracy_reward/std": 0.40537430346012115, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 799 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33258928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 851.7232360839844, "completions/mean_terminated_length": 770.7374114990234, "completions/min_length": 370.75, "completions/min_terminated_length": 370.75, "epoch": 0.23896647001717572, "grad_norm": 0.28782981634140015, "kl": 0.334228515625, "learning_rate": 1.8665703038206503e-05, "loss": 0.0269, "num_tokens": 407087894.0, "reward": 0.6953125298023224, "reward_std": 0.17428606003522873, "rewards/accuracy_reward/mean": 0.1986607164144516, "rewards/accuracy_reward/std": 0.3731900751590729, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 800 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 885.4553985595703, "completions/mean_terminated_length": 823.9922027587891, "completions/min_length": 351.25, "completions/min_terminated_length": 351.25, "epoch": 0.2392651781046972, "grad_norm": 0.2018183320760727, "kl": 0.44091796875, "learning_rate": 1.866025403784439e-05, "loss": 0.0296, "num_tokens": 407556658.0, "reward": 0.6277901977300644, "reward_std": 0.11426893435418606, "rewards/accuracy_reward/mean": 0.13169642724096775, "rewards/accuracy_reward/std": 0.3209918849170208, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 801 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 868.1652221679688, "completions/mean_terminated_length": 782.4070892333984, "completions/min_length": 360.25, "completions/min_terminated_length": 360.25, "epoch": 0.23956388619221866, "grad_norm": 0.19582675397396088, "kl": 0.2867431640625, "learning_rate": 1.8654794732544857e-05, "loss": 0.0206, "num_tokens": 408022508.0, "reward": 0.6456473618745804, "reward_std": 0.1343568991869688, "rewards/accuracy_reward/mean": 0.14732143003493547, "rewards/accuracy_reward/std": 0.33614304661750793, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.01421990292146802, "step": 802 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 874.4911193847656, "completions/mean_terminated_length": 794.4308013916016, "completions/min_length": 469.0, "completions/min_terminated_length": 469.0, "epoch": 0.23986259427974013, "grad_norm": 0.13113565742969513, "kl": 0.309814453125, "learning_rate": 1.8649325128804007e-05, "loss": 0.0174, "num_tokens": 408485512.0, "reward": 0.633370578289032, "reward_std": 0.0914585655555129, "rewards/accuracy_reward/mean": 0.13392857112921774, "rewards/accuracy_reward/std": 0.29282102547585964, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 803 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 886.5714721679688, "completions/mean_terminated_length": 807.6230163574219, "completions/min_length": 392.25, "completions/min_terminated_length": 392.25, "epoch": 0.2401613023672616, "grad_norm": 0.11032712459564209, "kl": 0.217529296875, "learning_rate": 1.8643845233130175e-05, "loss": 0.0146, "num_tokens": 408951992.0, "reward": 0.6183036118745804, "reward_std": 0.15332964062690735, "rewards/accuracy_reward/mean": 0.11830357182770967, "rewards/accuracy_reward/std": 0.31245772540569305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.5, "rewards/tag_count_reward/std": 0.0, "step": 804 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 866.0335235595703, "completions/mean_terminated_length": 798.2905731201172, "completions/min_length": 343.25, "completions/min_terminated_length": 343.25, "epoch": 0.24046001045478307, "grad_norm": 0.11301188915967941, "kl": 0.198486328125, "learning_rate": 1.863835505204396e-05, "loss": 0.0103, "num_tokens": 409407399.0, "reward": 0.695870578289032, "reward_std": 0.13154050335288048, "rewards/accuracy_reward/mean": 0.1964285671710968, "rewards/accuracy_reward/std": 0.37566692382097244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 805 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 889.7902069091797, "completions/mean_terminated_length": 794.0631103515625, "completions/min_length": 342.75, "completions/min_terminated_length": 342.75, "epoch": 0.24075871854230455, "grad_norm": 0.14512069523334503, "kl": 0.308349609375, "learning_rate": 1.8632854592078185e-05, "loss": 0.0121, "num_tokens": 409883433.0, "reward": 0.753348246216774, "reward_std": 0.1787901110947132, "rewards/accuracy_reward/mean": 0.2674851231276989, "rewards/accuracy_reward/std": 0.438255675137043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 806 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 860.8772735595703, "completions/mean_terminated_length": 780.6207275390625, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "epoch": 0.24105742662982602, "grad_norm": 0.1317783147096634, "kl": 0.322509765625, "learning_rate": 1.862734385977792e-05, "loss": 0.0199, "num_tokens": 410340162.0, "reward": 0.6869420111179352, "reward_std": 0.1141684539616108, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3820298761129379, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4994419664144516, "rewards/tag_count_reward/std": 0.005905694793909788, "step": 807 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 885.9888763427734, "completions/mean_terminated_length": 786.46826171875, "completions/min_length": 452.25, "completions/min_terminated_length": 452.25, "epoch": 0.24135613471734746, "grad_norm": 0.28908637166023254, "kl": 0.298583984375, "learning_rate": 1.8621822861700446e-05, "loss": 0.0161, "num_tokens": 410815133.0, "reward": 0.651785746216774, "reward_std": 0.12185943778604269, "rewards/accuracy_reward/mean": 0.15178571362048388, "rewards/accuracy_reward/std": 0.3310561180114746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.5, "rewards/tag_count_reward/std": 0.0, "step": 808 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 872.8370971679688, "completions/mean_terminated_length": 785.4023284912109, "completions/min_length": 352.5, "completions/min_terminated_length": 352.5, "epoch": 0.24165484280486893, "grad_norm": 0.2176218032836914, "kl": 0.4169921875, "learning_rate": 1.861629160441526e-05, "loss": 0.0229, "num_tokens": 411281508.0, "reward": 0.5926339626312256, "reward_std": 0.11431635729968548, "rewards/accuracy_reward/mean": 0.09375000279396772, "rewards/accuracy_reward/std": 0.2837582379579544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 809 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37053571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 880.9263916015625, "completions/mean_terminated_length": 801.5877532958984, "completions/min_length": 336.75, "completions/min_terminated_length": 336.75, "epoch": 0.2419535508923904, "grad_norm": 0.17922262847423553, "kl": 0.502685546875, "learning_rate": 1.8610750094504074e-05, "loss": 0.0276, "num_tokens": 411742579.0, "reward": 0.6311384290456772, "reward_std": 0.16386916860938072, "rewards/accuracy_reward/mean": 0.13392857182770967, "rewards/accuracy_reward/std": 0.32445572316646576, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 810 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 862.2745971679688, "completions/mean_terminated_length": 774.1748809814453, "completions/min_length": 327.25, "completions/min_terminated_length": 327.25, "epoch": 0.24225225897991187, "grad_norm": 0.08497980982065201, "kl": 0.173828125, "learning_rate": 1.860519833856079e-05, "loss": -0.0073, "num_tokens": 412195886.0, "reward": 0.7176339775323868, "reward_std": 0.13796889409422874, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.4048349857330322, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839253783226, "rewards/tag_count_reward/std": 0.008314208127558231, "step": 811 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 879.6674499511719, "completions/mean_terminated_length": 779.8072509765625, "completions/min_length": 303.75, "completions/min_terminated_length": 303.75, "epoch": 0.24255096706743334, "grad_norm": 0.13106849789619446, "kl": 0.352783203125, "learning_rate": 1.8599636343191515e-05, "loss": 0.0205, "num_tokens": 412660137.0, "reward": 0.6785714626312256, "reward_std": 0.15104380436241627, "rewards/accuracy_reward/mean": 0.1808035704307258, "rewards/accuracy_reward/std": 0.3564932718873024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 812 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 933.7723541259766, "completions/mean_terminated_length": 816.3617248535156, "completions/min_length": 420.5, "completions/min_terminated_length": 420.5, "epoch": 0.2428496751549548, "grad_norm": 0.2811434864997864, "kl": 0.55908203125, "learning_rate": 1.8594064115014528e-05, "loss": 0.0166, "num_tokens": 413150163.0, "reward": 0.664620578289032, "reward_std": 0.1424594633281231, "rewards/accuracy_reward/mean": 0.17819940485060215, "rewards/accuracy_reward/std": 0.37358642369508743, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 813 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 924.8348541259766, "completions/mean_terminated_length": 829.9844055175781, "completions/min_length": 405.25, "completions/min_terminated_length": 405.25, "epoch": 0.24314838324247628, "grad_norm": 0.14968731999397278, "kl": 0.298095703125, "learning_rate": 1.858848166066029e-05, "loss": 0.0115, "num_tokens": 413636057.0, "reward": 0.6902901977300644, "reward_std": 0.1699742991477251, "rewards/accuracy_reward/mean": 0.19196428172290325, "rewards/accuracy_reward/std": 0.38406533002853394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258992433548, "rewards/tag_count_reward/std": 0.017717084381729364, "step": 814 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 906.3750457763672, "completions/mean_terminated_length": 800.7249603271484, "completions/min_length": 458.75, "completions/min_terminated_length": 458.75, "epoch": 0.24344709132999776, "grad_norm": 0.3251839280128479, "kl": 0.60693359375, "learning_rate": 1.8582888986771423e-05, "loss": 0.0359, "num_tokens": 414111601.0, "reward": 0.6110491305589676, "reward_std": 0.12697431119158864, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.25317634269595146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.494977667927742, "rewards/tag_count_reward/std": 0.03507901635020971, "step": 815 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 902.5245971679688, "completions/mean_terminated_length": 793.0145263671875, "completions/min_length": 348.5, "completions/min_terminated_length": 348.5, "epoch": 0.24374579941751923, "grad_norm": 0.13688452541828156, "kl": 0.41552734375, "learning_rate": 1.8577286100002723e-05, "loss": 0.018, "num_tokens": 414588908.0, "reward": 0.577566996216774, "reward_std": 0.04546040389686823, "rewards/accuracy_reward/mean": 0.08035714412108064, "rewards/accuracy_reward/std": 0.22745841182768345, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 816 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 939.888427734375, "completions/mean_terminated_length": 835.7590484619141, "completions/min_length": 497.0, "completions/min_terminated_length": 497.0, "epoch": 0.2440445075050407, "grad_norm": 0.3062634766101837, "kl": 0.52294921875, "learning_rate": 1.8571673007021124e-05, "loss": 0.0246, "num_tokens": 415077386.0, "reward": 0.6222098469734192, "reward_std": 0.12232430651783943, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.3302958086133003, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 817 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 969.3103179931641, "completions/mean_terminated_length": 837.7853851318359, "completions/min_length": 301.75, "completions/min_terminated_length": 301.75, "epoch": 0.24434321559256217, "grad_norm": 0.3969358205795288, "kl": 0.7958984375, "learning_rate": 1.8566049714505717e-05, "loss": 0.0227, "num_tokens": 415585541.0, "reward": 0.5518973618745804, "reward_std": 0.13905747327953577, "rewards/accuracy_reward/mean": 0.0602678582072258, "rewards/accuracy_reward/std": 0.20400601252913475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04508844017982483, "step": 818 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 948.0670166015625, "completions/mean_terminated_length": 833.896240234375, "completions/min_length": 376.75, "completions/min_terminated_length": 376.75, "epoch": 0.24464192368008364, "grad_norm": 0.2959538400173187, "kl": 1.1162109375, "learning_rate": 1.8560416229147718e-05, "loss": 0.0425, "num_tokens": 416075715.0, "reward": 0.5792410969734192, "reward_std": 0.1360208559781313, "rewards/accuracy_reward/mean": 0.09412202518433332, "rewards/accuracy_reward/std": 0.2828701287508011, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05265482049435377, "step": 819 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 955.8527221679688, "completions/mean_terminated_length": 855.4492034912109, "completions/min_length": 574.5, "completions/min_terminated_length": 574.5, "epoch": 0.2449406317676051, "grad_norm": 0.2997978627681732, "kl": 1.369140625, "learning_rate": 1.8554772557650474e-05, "loss": 0.0706, "num_tokens": 416576209.0, "reward": 0.6830357313156128, "reward_std": 0.17038046196103096, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.3912891671061516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194884091616, "step": 820 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 933.4531707763672, "completions/mean_terminated_length": 818.6847839355469, "completions/min_length": 419.75, "completions/min_terminated_length": 419.75, "epoch": 0.24523933985512658, "grad_norm": 0.6604508757591248, "kl": 2.248046875, "learning_rate": 1.854911870672947e-05, "loss": 0.0892, "num_tokens": 417067756.0, "reward": 0.6729910969734192, "reward_std": 0.21328434348106384, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.34811433404684067, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058666424825787544, "step": 821 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 944.5580749511719, "completions/mean_terminated_length": 830.4053497314453, "completions/min_length": 458.25, "completions/min_terminated_length": 458.25, "epoch": 0.24553804794264805, "grad_norm": 0.5736284852027893, "kl": 2.28515625, "learning_rate": 1.8543454683112272e-05, "loss": 0.093, "num_tokens": 417559830.0, "reward": 0.5781250298023224, "reward_std": 0.1451370157301426, "rewards/accuracy_reward/mean": 0.09598214458674192, "rewards/accuracy_reward/std": 0.27311623841524124, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06270094495266676, "step": 822 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 924.216552734375, "completions/mean_terminated_length": 784.7661590576172, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "epoch": 0.24583675603016952, "grad_norm": 0.341799259185791, "kl": 2.474609375, "learning_rate": 1.8537780493538576e-05, "loss": 0.1069, "num_tokens": 418040807.0, "reward": 0.5736607313156128, "reward_std": 0.16589340567588806, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.28811274468898773, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4754464328289032, "rewards/tag_count_reward/std": 0.07066364586353302, "step": 823 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7075892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 966.3772735595703, "completions/mean_terminated_length": 826.8502655029297, "completions/min_length": 520.0, "completions/min_terminated_length": 520.0, "epoch": 0.246135464117691, "grad_norm": 0.4600703716278076, "kl": 1.912109375, "learning_rate": 1.853209614476017e-05, "loss": 0.0817, "num_tokens": 418539376.0, "reward": 0.6830357313156128, "reward_std": 0.22321095690131187, "rewards/accuracy_reward/mean": 0.2098214291036129, "rewards/accuracy_reward/std": 0.39490821212530136, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.473214291036129, "rewards/tag_count_reward/std": 0.07738563418388367, "step": 824 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 958.0625305175781, "completions/mean_terminated_length": 814.596923828125, "completions/min_length": 381.25, "completions/min_terminated_length": 381.25, "epoch": 0.24643417220521247, "grad_norm": 0.5884599685668945, "kl": 2.017578125, "learning_rate": 1.8526401643540924e-05, "loss": 0.0888, "num_tokens": 419033212.0, "reward": 0.5172991305589676, "reward_std": 0.11945973336696625, "rewards/accuracy_reward/mean": 0.044642857974395156, "rewards/accuracy_reward/std": 0.1609747651964426, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.47265625, "rewards/tag_count_reward/std": 0.07776896096765995, "step": 825 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 955.0111999511719, "completions/mean_terminated_length": 837.4423522949219, "completions/min_length": 472.0, "completions/min_terminated_length": 472.0, "epoch": 0.24673288029273394, "grad_norm": 0.2668839395046234, "kl": 1.8828125, "learning_rate": 1.8520696996656787e-05, "loss": 0.0857, "num_tokens": 419526737.0, "reward": 0.5781250149011612, "reward_std": 0.1677851751446724, "rewards/accuracy_reward/mean": 0.09598214272409678, "rewards/accuracy_reward/std": 0.28744081407785416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428507566452, "rewards/tag_count_reward/std": 0.06430223677307367, "step": 826 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7120535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 964.1495971679688, "completions/mean_terminated_length": 817.7650299072266, "completions/min_length": 324.25, "completions/min_terminated_length": 324.25, "epoch": 0.2470315883802554, "grad_norm": 0.34844154119491577, "kl": 3.53515625, "learning_rate": 1.851498221089579e-05, "loss": 0.1466, "num_tokens": 420038212.0, "reward": 0.5362723469734192, "reward_std": 0.1678325291723013, "rewards/accuracy_reward/mean": 0.06919642817229033, "rewards/accuracy_reward/std": 0.2136813923716545, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4670758917927742, "rewards/tag_count_reward/std": 0.08449990302324295, "step": 827 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 942.9933471679688, "completions/mean_terminated_length": 811.9663543701172, "completions/min_length": 481.25, "completions/min_terminated_length": 481.25, "epoch": 0.24733029646777688, "grad_norm": 0.4089573919773102, "kl": 2.45703125, "learning_rate": 1.8509257293058023e-05, "loss": 0.1105, "num_tokens": 420525761.0, "reward": 0.6445312798023224, "reward_std": 0.16354883834719658, "rewards/accuracy_reward/mean": 0.16294642677530646, "rewards/accuracy_reward/std": 0.3362807258963585, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06409717723727226, "step": 828 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6607142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 957.9040679931641, "completions/mean_terminated_length": 829.7921905517578, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.24762900455529832, "grad_norm": 0.3160850405693054, "kl": 2.5, "learning_rate": 1.850352224995563e-05, "loss": 0.0897, "num_tokens": 421034230.0, "reward": 0.522879496216774, "reward_std": 0.12701672036200762, "rewards/accuracy_reward/mean": 0.04910714412108064, "rewards/accuracy_reward/std": 0.1434241496026516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4737723246216774, "rewards/tag_count_reward/std": 0.0766516886651516, "step": 829 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 942.763427734375, "completions/mean_terminated_length": 824.2177429199219, "completions/min_length": 404.25, "completions/min_terminated_length": 404.25, "epoch": 0.2479277126428198, "grad_norm": 0.3634090721607208, "kl": 2.109375, "learning_rate": 1.849777708841281e-05, "loss": 0.0886, "num_tokens": 421535772.0, "reward": 0.510044664144516, "reward_std": 0.12350511364638805, "rewards/accuracy_reward/mean": 0.03125000046566129, "rewards/accuracy_reward/std": 0.14895273000001907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946343421936, "rewards/tag_count_reward/std": 0.06969121471047401, "step": 830 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 946.9777069091797, "completions/mean_terminated_length": 844.3260040283203, "completions/min_length": 475.0, "completions/min_terminated_length": 475.0, "epoch": 0.24822642073034126, "grad_norm": 0.30570411682128906, "kl": 1.953125, "learning_rate": 1.849202181526579e-05, "loss": 0.0868, "num_tokens": 422033234.0, "reward": 0.5435268059372902, "reward_std": 0.1257669422775507, "rewards/accuracy_reward/mean": 0.06696428498253226, "rewards/accuracy_reward/std": 0.20403443276882172, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4765624925494194, "rewards/tag_count_reward/std": 0.07279855012893677, "step": 831 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 939.9978179931641, "completions/mean_terminated_length": 811.3138275146484, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "epoch": 0.24852512881786273, "grad_norm": 0.5623048543930054, "kl": 1.48046875, "learning_rate": 1.8486256437362842e-05, "loss": 0.0636, "num_tokens": 422526769.0, "reward": 0.5362723395228386, "reward_std": 0.11584782041609287, "rewards/accuracy_reward/mean": 0.05357142840512097, "rewards/accuracy_reward/std": 0.17494087480008602, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06354017928242683, "step": 832 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6495535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 962.0647888183594, "completions/mean_terminated_length": 847.4921112060547, "completions/min_length": 340.25, "completions/min_terminated_length": 340.25, "epoch": 0.2488238369053842, "grad_norm": 0.2080027312040329, "kl": 1.796875, "learning_rate": 1.848048096156426e-05, "loss": 0.0714, "num_tokens": 423030510.0, "reward": 0.5937500149011612, "reward_std": 0.1483457311987877, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.25846807286143303, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.05946851149201393, "step": 833 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 951.8170166015625, "completions/mean_terminated_length": 829.0874176025391, "completions/min_length": 441.0, "completions/min_terminated_length": 441.0, "epoch": 0.24912254499290568, "grad_norm": 0.29027655720710754, "kl": 1.84375, "learning_rate": 1.8474695394742345e-05, "loss": 0.0713, "num_tokens": 423525020.0, "reward": 0.5809152200818062, "reward_std": 0.11369989812374115, "rewards/accuracy_reward/mean": 0.09375, "rewards/accuracy_reward/std": 0.24249250069260597, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05331833194941282, "step": 834 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6138392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 948.8303985595703, "completions/mean_terminated_length": 830.8705749511719, "completions/min_length": 530.25, "completions/min_terminated_length": 530.25, "epoch": 0.24942125308042715, "grad_norm": 0.42673027515411377, "kl": 1.892578125, "learning_rate": 1.8468899743781416e-05, "loss": 0.0813, "num_tokens": 424025968.0, "reward": 0.599888414144516, "reward_std": 0.10825466318055987, "rewards/accuracy_reward/mean": 0.1071428544819355, "rewards/accuracy_reward/std": 0.20425515621900558, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 835 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 914.2121124267578, "completions/mean_terminated_length": 792.5015106201172, "completions/min_length": 449.75, "completions/min_terminated_length": 449.75, "epoch": 0.24971996116794862, "grad_norm": 0.7097676992416382, "kl": 1.64453125, "learning_rate": 1.8463094015577772e-05, "loss": 0.0688, "num_tokens": 424500495.0, "reward": 0.5937500298023224, "reward_std": 0.12604269059374928, "rewards/accuracy_reward/mean": 0.10267857275903225, "rewards/accuracy_reward/std": 0.22855757176876068, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.0447555473074317, "step": 836 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 910.3683471679688, "completions/mean_terminated_length": 773.6255493164062, "completions/min_length": 463.25, "completions/min_terminated_length": 463.25, "epoch": 0.2500186692554701, "grad_norm": 0.2511723041534424, "kl": 1.44921875, "learning_rate": 1.8457278217039735e-05, "loss": 0.0632, "num_tokens": 424980452.0, "reward": 0.6679687798023224, "reward_std": 0.13755724765360355, "rewards/accuracy_reward/mean": 0.17633928917348385, "rewards/accuracy_reward/std": 0.36573970317840576, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.04348720656707883, "step": 837 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 957.1272888183594, "completions/mean_terminated_length": 843.8347473144531, "completions/min_length": 510.75, "completions/min_terminated_length": 510.75, "epoch": 0.2503173773429916, "grad_norm": 0.36748674511909485, "kl": 1.0927734375, "learning_rate": 1.845145235508758e-05, "loss": 0.0523, "num_tokens": 425489709.0, "reward": 0.6316964626312256, "reward_std": 0.12905122805386782, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3450803607702255, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03495204821228981, "step": 838 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 954.3147735595703, "completions/mean_terminated_length": 818.3227081298828, "completions/min_length": 333.75, "completions/min_terminated_length": 333.75, "epoch": 0.25061608543051306, "grad_norm": 0.23349829018115997, "kl": 1.2236328125, "learning_rate": 1.8445616436653567e-05, "loss": 0.0496, "num_tokens": 425993658.0, "reward": 0.5128348469734192, "reward_std": 0.08225569874048233, "rewards/accuracy_reward/mean": 0.022321429336443543, "rewards/accuracy_reward/std": 0.14199810102581978, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046577731147408485, "step": 839 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 920.6094207763672, "completions/mean_terminated_length": 808.9716796875, "completions/min_length": 383.25, "completions/min_terminated_length": 383.25, "epoch": 0.2509147935180345, "grad_norm": 0.12848585844039917, "kl": 0.68701171875, "learning_rate": 1.8439770468681934e-05, "loss": 0.0409, "num_tokens": 426478619.0, "reward": 0.661272332072258, "reward_std": 0.15736837801523507, "rewards/accuracy_reward/mean": 0.1651785671710968, "rewards/accuracy_reward/std": 0.30341094732284546, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 840 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6919642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 972.7634429931641, "completions/mean_terminated_length": 862.1879730224609, "completions/min_length": 484.25, "completions/min_terminated_length": 484.25, "epoch": 0.25121350160555594, "grad_norm": 0.43986770510673523, "kl": 0.65185546875, "learning_rate": 1.843391445812886e-05, "loss": 0.0266, "num_tokens": 426988849.0, "reward": 0.6266741305589676, "reward_std": 0.14745336771011353, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.28138359636068344, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 841 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 933.8727874755859, "completions/mean_terminated_length": 813.6678924560547, "completions/min_length": 488.5, "completions/min_terminated_length": 488.5, "epoch": 0.2515122096930774, "grad_norm": 0.40314948558807373, "kl": 0.8134765625, "learning_rate": 1.8428048411962475e-05, "loss": 0.0476, "num_tokens": 427473608.0, "reward": 0.704799123108387, "reward_std": 0.13540798984467983, "rewards/accuracy_reward/mean": 0.2354910708963871, "rewards/accuracy_reward/std": 0.3119966834783554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 842 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 909.4040679931641, "completions/mean_terminated_length": 796.96484375, "completions/min_length": 465.5, "completions/min_terminated_length": 465.5, "epoch": 0.2518109177805989, "grad_norm": 0.2349177449941635, "kl": 0.449462890625, "learning_rate": 1.8422172337162865e-05, "loss": 0.0289, "num_tokens": 427946205.0, "reward": 0.6188616305589676, "reward_std": 0.062400735914707184, "rewards/accuracy_reward/mean": 0.12276785518042743, "rewards/accuracy_reward/std": 0.2733932565897703, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 843 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 939.2388763427734, "completions/mean_terminated_length": 822.4857788085938, "completions/min_length": 542.0, "completions/min_terminated_length": 542.0, "epoch": 0.25210962586812036, "grad_norm": 0.23189488053321838, "kl": 0.83203125, "learning_rate": 1.8416286240722037e-05, "loss": 0.0463, "num_tokens": 428442248.0, "reward": 0.671316996216774, "reward_std": 0.17530683614313602, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.3471510782837868, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.033598463982343674, "step": 844 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 915.4062805175781, "completions/mean_terminated_length": 800.6313323974609, "completions/min_length": 413.5, "completions/min_terminated_length": 413.5, "epoch": 0.25240833395564183, "grad_norm": 0.18566618859767914, "kl": 1.185546875, "learning_rate": 1.8410390129643927e-05, "loss": 0.0623, "num_tokens": 428921614.0, "reward": 0.623325914144516, "reward_std": 0.11845726519823074, "rewards/accuracy_reward/mean": 0.13169643026776612, "rewards/accuracy_reward/std": 0.30792444571852684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 845 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 906.6897583007812, "completions/mean_terminated_length": 802.3415832519531, "completions/min_length": 494.0, "completions/min_terminated_length": 494.0, "epoch": 0.2527070420431633, "grad_norm": 0.2120666801929474, "kl": 1.0859375, "learning_rate": 1.840448401094438e-05, "loss": 0.0552, "num_tokens": 429397891.0, "reward": 0.6551339775323868, "reward_std": 0.18672745302319527, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.3628467172384262, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 846 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 894.6942291259766, "completions/mean_terminated_length": 779.0514831542969, "completions/min_length": 340.75, "completions/min_terminated_length": 340.75, "epoch": 0.25300575013068477, "grad_norm": 0.44251546263694763, "kl": 1.564453125, "learning_rate": 1.8398567891651163e-05, "loss": 0.0711, "num_tokens": 429871834.0, "reward": 0.752232164144516, "reward_std": 0.1349023450165987, "rewards/accuracy_reward/mean": 0.2589285671710968, "rewards/accuracy_reward/std": 0.4285278990864754, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 847 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 943.2902221679688, "completions/mean_terminated_length": 838.1013336181641, "completions/min_length": 513.75, "completions/min_terminated_length": 513.75, "epoch": 0.25330445821820624, "grad_norm": 0.48596423864364624, "kl": 1.783203125, "learning_rate": 1.8392641778803935e-05, "loss": 0.0714, "num_tokens": 430373980.0, "reward": 0.5825893133878708, "reward_std": 0.07888280786573887, "rewards/accuracy_reward/mean": 0.09151785937137902, "rewards/accuracy_reward/std": 0.21517368033528328, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.044314838480204344, "step": 848 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49553571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 916.1250305175781, "completions/mean_terminated_length": 818.0095367431641, "completions/min_length": 370.5, "completions/min_terminated_length": 370.5, "epoch": 0.2536031663057277, "grad_norm": 0.3698837161064148, "kl": 1.7890625, "learning_rate": 1.8386705679454243e-05, "loss": 0.0555, "num_tokens": 430857668.0, "reward": 0.6149553954601288, "reward_std": 0.13868924230337143, "rewards/accuracy_reward/mean": 0.12499999906867743, "rewards/accuracy_reward/std": 0.3110002279281616, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04809202253818512, "step": 849 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46205357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 893.7656555175781, "completions/mean_terminated_length": 786.2413635253906, "completions/min_length": 448.25, "completions/min_terminated_length": 448.25, "epoch": 0.2539018743932492, "grad_norm": 0.15074221789836884, "kl": 1.265625, "learning_rate": 1.838075960066552e-05, "loss": 0.0616, "num_tokens": 431327243.0, "reward": 0.5837053954601288, "reward_std": 0.12924108654260635, "rewards/accuracy_reward/mean": 0.09375000116415322, "rewards/accuracy_reward/std": 0.2646229788661003, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046074194367975, "step": 850 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 892.9241485595703, "completions/mean_terminated_length": 798.1752777099609, "completions/min_length": 468.75, "completions/min_terminated_length": 468.75, "epoch": 0.25420058248077065, "grad_norm": 0.15105213224887848, "kl": 1.0498046875, "learning_rate": 1.837480354951308e-05, "loss": 0.0614, "num_tokens": 431800233.0, "reward": 0.6579241305589676, "reward_std": 0.17739460058510303, "rewards/accuracy_reward/mean": 0.16517857182770967, "rewards/accuracy_reward/std": 0.3513757437467575, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196588039398, "step": 851 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 906.5803833007812, "completions/mean_terminated_length": 810.4095153808594, "completions/min_length": 533.5, "completions/min_terminated_length": 533.5, "epoch": 0.2544992905682921, "grad_norm": 0.2857784628868103, "kl": 1.138671875, "learning_rate": 1.8368837533084092e-05, "loss": 0.0644, "num_tokens": 432276813.0, "reward": 0.6713169887661934, "reward_std": 0.15548076387494802, "rewards/accuracy_reward/mean": 0.1808035708963871, "rewards/accuracy_reward/std": 0.31106823682785034, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.040530065074563026, "step": 852 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 906.0870971679688, "completions/mean_terminated_length": 821.7002410888672, "completions/min_length": 463.5, "completions/min_terminated_length": 463.5, "epoch": 0.2547979986558136, "grad_norm": 0.353282630443573, "kl": 1.3515625, "learning_rate": 1.8362861558477597e-05, "loss": 0.0867, "num_tokens": 432752276.0, "reward": 0.7237723469734192, "reward_std": 0.16901731863617897, "rewards/accuracy_reward/mean": 0.2366071455180645, "rewards/accuracy_reward/std": 0.42195069789886475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 853 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 916.4866485595703, "completions/mean_terminated_length": 810.8163452148438, "completions/min_length": 415.25, "completions/min_terminated_length": 415.25, "epoch": 0.25509670674333507, "grad_norm": 0.24270249903202057, "kl": 1.15625, "learning_rate": 1.835687563280447e-05, "loss": 0.05, "num_tokens": 433241118.0, "reward": 0.5597098469734192, "reward_std": 0.1299648992717266, "rewards/accuracy_reward/mean": 0.0691964307334274, "rewards/accuracy_reward/std": 0.21221555583178997, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04597289999946952, "step": 854 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 904.1094055175781, "completions/mean_terminated_length": 815.2723693847656, "completions/min_length": 516.75, "completions/min_terminated_length": 516.75, "epoch": 0.25539541483085654, "grad_norm": 0.31991881132125854, "kl": 1.3740234375, "learning_rate": 1.8350879763187433e-05, "loss": 0.0721, "num_tokens": 433716143.0, "reward": 0.631138414144516, "reward_std": 0.15076376497745514, "rewards/accuracy_reward/mean": 0.14285713946446776, "rewards/accuracy_reward/std": 0.3267778977751732, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 855 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 878.622802734375, "completions/mean_terminated_length": 795.7445678710938, "completions/min_length": 438.75, "completions/min_terminated_length": 438.75, "epoch": 0.255694122918378, "grad_norm": 0.4117283523082733, "kl": 0.9990234375, "learning_rate": 1.8344873956761045e-05, "loss": 0.0386, "num_tokens": 434191174.0, "reward": 0.611607164144516, "reward_std": 0.13545201905071735, "rewards/accuracy_reward/mean": 0.11607142956927419, "rewards/accuracy_reward/std": 0.2939666397869587, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 856 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 869.2053833007812, "completions/mean_terminated_length": 785.3972778320312, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "epoch": 0.2559928310058995, "grad_norm": 0.2701517343521118, "kl": 1.669921875, "learning_rate": 1.8338858220671683e-05, "loss": 0.075, "num_tokens": 434649442.0, "reward": 0.5747768133878708, "reward_std": 0.1430507991462946, "rewards/accuracy_reward/mean": 0.08705357415601611, "rewards/accuracy_reward/std": 0.25784094259142876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 857 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 906.8594055175781, "completions/mean_terminated_length": 789.7210845947266, "completions/min_length": 481.25, "completions/min_terminated_length": 481.25, "epoch": 0.25629153909342095, "grad_norm": 0.27541056275367737, "kl": 1.3095703125, "learning_rate": 1.8332832562077544e-05, "loss": 0.0655, "num_tokens": 435131267.0, "reward": 0.603236623108387, "reward_std": 0.13293192628771067, "rewards/accuracy_reward/mean": 0.11383928544819355, "rewards/accuracy_reward/std": 0.25197184458374977, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 858 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 904.5625305175781, "completions/mean_terminated_length": 795.1398315429688, "completions/min_length": 477.25, "completions/min_terminated_length": 477.25, "epoch": 0.2565902471809424, "grad_norm": 0.22350642085075378, "kl": 1.8046875, "learning_rate": 1.8326796988148627e-05, "loss": 0.0852, "num_tokens": 435613551.0, "reward": 0.6099330633878708, "reward_std": 0.1896454393863678, "rewards/accuracy_reward/mean": 0.1227678544819355, "rewards/accuracy_reward/std": 0.3270955830812454, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055374542251229286, "step": 859 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 902.3906707763672, "completions/mean_terminated_length": 807.3505249023438, "completions/min_length": 447.25, "completions/min_terminated_length": 447.25, "epoch": 0.2568889552684639, "grad_norm": 0.22896935045719147, "kl": 1.85546875, "learning_rate": 1.8320751506066738e-05, "loss": 0.0791, "num_tokens": 436085598.0, "reward": 0.655133955180645, "reward_std": 0.1568406606093049, "rewards/accuracy_reward/mean": 0.16964285261929035, "rewards/accuracy_reward/std": 0.29758892953395844, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.058124168775975704, "step": 860 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6540178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 960.9219207763672, "completions/mean_terminated_length": 857.0658111572266, "completions/min_length": 587.0, "completions/min_terminated_length": 587.0, "epoch": 0.25718766335598536, "grad_norm": 0.1874084174633026, "kl": 1.2509765625, "learning_rate": 1.8314696123025456e-05, "loss": 0.0571, "num_tokens": 436592699.0, "reward": 0.580357164144516, "reward_std": 0.18099823221564293, "rewards/accuracy_reward/mean": 0.09151785634458065, "rewards/accuracy_reward/std": 0.2836172953248024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051120287738740444, "step": 861 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 931.6986999511719, "completions/mean_terminated_length": 822.7330474853516, "completions/min_length": 480.25, "completions/min_terminated_length": 480.25, "epoch": 0.25748637144350683, "grad_norm": 0.41709259152412415, "kl": 1.560546875, "learning_rate": 1.8308630846230158e-05, "loss": 0.0813, "num_tokens": 437088212.0, "reward": 0.597098246216774, "reward_std": 0.121142846532166, "rewards/accuracy_reward/mean": 0.11383928777649999, "rewards/accuracy_reward/std": 0.29681966081261635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.061202285811305046, "step": 862 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 923.9152069091797, "completions/mean_terminated_length": 804.8315887451172, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "epoch": 0.2577850795310283, "grad_norm": 0.17245742678642273, "kl": 1.474609375, "learning_rate": 1.8302555682897986e-05, "loss": 0.0725, "num_tokens": 437567758.0, "reward": 0.6517857313156128, "reward_std": 0.17791668139398098, "rewards/accuracy_reward/mean": 0.16517856903374195, "rewards/accuracy_reward/std": 0.3685881793498993, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05632450245320797, "step": 863 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 943.5625457763672, "completions/mean_terminated_length": 841.4161682128906, "completions/min_length": 467.75, "completions/min_terminated_length": 467.75, "epoch": 0.2580837876185498, "grad_norm": 0.25581321120262146, "kl": 2.703125, "learning_rate": 1.8296470640257854e-05, "loss": 0.135, "num_tokens": 438062010.0, "reward": 0.6233259290456772, "reward_std": 0.17576447501778603, "rewards/accuracy_reward/mean": 0.14918154664337635, "rewards/accuracy_reward/std": 0.32962777838110924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366156578064, "rewards/tag_count_reward/std": 0.07043877243995667, "step": 864 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.65625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 960.747802734375, "completions/mean_terminated_length": 841.8749694824219, "completions/min_length": 496.0, "completions/min_terminated_length": 496.0, "epoch": 0.25838249570607125, "grad_norm": 0.3236844837665558, "kl": 1.98828125, "learning_rate": 1.8290375725550417e-05, "loss": 0.0882, "num_tokens": 438566761.0, "reward": 0.5809152126312256, "reward_std": 0.14806815469637513, "rewards/accuracy_reward/mean": 0.09598214295692742, "rewards/accuracy_reward/std": 0.2196560762822628, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05540649499744177, "step": 865 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6808035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.25, "completions/mean_length": 968.1295013427734, "completions/mean_terminated_length": 850.3198852539062, "completions/min_length": 480.5, "completions/min_terminated_length": 480.5, "epoch": 0.2586812037935927, "grad_norm": 0.31682896614074707, "kl": 2.03125, "learning_rate": 1.8284270946028092e-05, "loss": 0.0802, "num_tokens": 439077379.0, "reward": 0.684151828289032, "reward_std": 0.23317820951342583, "rewards/accuracy_reward/mean": 0.19866071827709675, "rewards/accuracy_reward/std": 0.38095951825380325, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058050588704645634, "step": 866 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 921.8973541259766, "completions/mean_terminated_length": 789.5749664306641, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "epoch": 0.2589799118811142, "grad_norm": 0.24991296231746674, "kl": 2.115234375, "learning_rate": 1.8278156308955024e-05, "loss": 0.097, "num_tokens": 439556373.0, "reward": 0.6149553954601288, "reward_std": 0.1618126519024372, "rewards/accuracy_reward/mean": 0.13095237966626883, "rewards/accuracy_reward/std": 0.2812983766198158, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05821291171014309, "step": 867 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6919642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 970.3906555175781, "completions/mean_terminated_length": 855.7615814208984, "completions/min_length": 584.75, "completions/min_terminated_length": 584.75, "epoch": 0.25927861996863566, "grad_norm": 0.34960758686065674, "kl": 2.36328125, "learning_rate": 1.8272031821607087e-05, "loss": 0.0989, "num_tokens": 440063844.0, "reward": 0.6110491305589676, "reward_std": 0.18178540095686913, "rewards/accuracy_reward/mean": 0.1272321455180645, "rewards/accuracy_reward/std": 0.32302237302064896, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.061596741899847984, "step": 868 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 957.9487152099609, "completions/mean_terminated_length": 865.0555267333984, "completions/min_length": 582.75, "completions/min_terminated_length": 582.75, "epoch": 0.25957732805615713, "grad_norm": 0.19258257746696472, "kl": 1.458984375, "learning_rate": 1.8265897491271885e-05, "loss": 0.0662, "num_tokens": 440569853.0, "reward": 0.6841518133878708, "reward_std": 0.13634956814348698, "rewards/accuracy_reward/mean": 0.19642857951112092, "rewards/accuracy_reward/std": 0.3200908284634352, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05369503889232874, "step": 869 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 959.6830902099609, "completions/mean_terminated_length": 835.0182342529297, "completions/min_length": 551.0, "completions/min_terminated_length": 551.0, "epoch": 0.2598760361436786, "grad_norm": 0.2768467366695404, "kl": 1.763671875, "learning_rate": 1.825975332524873e-05, "loss": 0.0744, "num_tokens": 441069775.0, "reward": 0.6032366305589676, "reward_std": 0.1436702348291874, "rewards/accuracy_reward/mean": 0.11607142630964518, "rewards/accuracy_reward/std": 0.2508578822016716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054504433646798134, "step": 870 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 947.685302734375, "completions/mean_terminated_length": 830.7784271240234, "completions/min_length": 474.75, "completions/min_terminated_length": 474.75, "epoch": 0.2601747442312001, "grad_norm": 0.3470250070095062, "kl": 1.958984375, "learning_rate": 1.8253599330848638e-05, "loss": 0.0829, "num_tokens": 441560658.0, "reward": 0.6523437649011612, "reward_std": 0.19292787462472916, "rewards/accuracy_reward/mean": 0.16964286100119352, "rewards/accuracy_reward/std": 0.3560854122042656, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008843421936, "rewards/tag_count_reward/std": 0.06217210926115513, "step": 871 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 947.5223693847656, "completions/mean_terminated_length": 839.6054077148438, "completions/min_length": 303.75, "completions/min_terminated_length": 303.75, "epoch": 0.26047345231872154, "grad_norm": 0.2717914879322052, "kl": 1.53125, "learning_rate": 1.8247435515394317e-05, "loss": 0.0625, "num_tokens": 442063068.0, "reward": 0.6512277126312256, "reward_std": 0.23464039340615273, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.35191813483834267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06139749940484762, "step": 872 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 951.7187957763672, "completions/mean_terminated_length": 841.2294769287109, "completions/min_length": 529.5, "completions/min_terminated_length": 529.5, "epoch": 0.260772160406243, "grad_norm": 0.28390640020370483, "kl": 1.2548828125, "learning_rate": 1.8241261886220155e-05, "loss": 0.0576, "num_tokens": 442555070.0, "reward": 0.5781250223517418, "reward_std": 0.13737516570836306, "rewards/accuracy_reward/mean": 0.09151785634458065, "rewards/accuracy_reward/std": 0.24243023246526718, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05536533612757921, "step": 873 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6964285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 966.3527221679688, "completions/mean_terminated_length": 839.388427734375, "completions/min_length": 527.25, "completions/min_terminated_length": 527.25, "epoch": 0.2610708684937645, "grad_norm": 0.3199367821216583, "kl": 1.548828125, "learning_rate": 1.8235078450672242e-05, "loss": 0.0676, "num_tokens": 443056860.0, "reward": 0.529575914144516, "reward_std": 0.14359750412404537, "rewards/accuracy_reward/mean": 0.04241071501746774, "rewards/accuracy_reward/std": 0.19719241559505463, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 874 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 922.7946624755859, "completions/mean_terminated_length": 816.0272979736328, "completions/min_length": 476.25, "completions/min_terminated_length": 476.25, "epoch": 0.26136957658128596, "grad_norm": 0.1891905963420868, "kl": 1.2177734375, "learning_rate": 1.8228885216108307e-05, "loss": 0.0537, "num_tokens": 443545328.0, "reward": 0.7198661118745804, "reward_std": 0.20376469939947128, "rewards/accuracy_reward/mean": 0.22991071362048388, "rewards/accuracy_reward/std": 0.3158479183912277, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.047033360693603754, "step": 875 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 970.4531707763672, "completions/mean_terminated_length": 854.2630920410156, "completions/min_length": 528.5, "completions/min_terminated_length": 528.5, "epoch": 0.2616682846688074, "grad_norm": 0.2695755958557129, "kl": 1.3564453125, "learning_rate": 1.822268218989775e-05, "loss": 0.055, "num_tokens": 444059147.0, "reward": 0.6244419813156128, "reward_std": 0.17495779879391193, "rewards/accuracy_reward/mean": 0.13392857182770967, "rewards/accuracy_reward/std": 0.32647764682769775, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 876 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6941964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 960.0960235595703, "completions/mean_terminated_length": 808.7906799316406, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "epoch": 0.2619669927563289, "grad_norm": 0.29240065813064575, "kl": 1.55078125, "learning_rate": 1.821646937942164e-05, "loss": 0.0635, "num_tokens": 444559270.0, "reward": 0.5982142984867096, "reward_std": 0.1353430673480034, "rewards/accuracy_reward/mean": 0.1164434514939785, "rewards/accuracy_reward/std": 0.30978530645370483, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05958512146025896, "step": 877 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 935.0580749511719, "completions/mean_terminated_length": 829.8091583251953, "completions/min_length": 487.25, "completions/min_terminated_length": 487.25, "epoch": 0.26226570084385037, "grad_norm": 0.3253512382507324, "kl": 1.240234375, "learning_rate": 1.8210246792072655e-05, "loss": 0.0489, "num_tokens": 445046672.0, "reward": 0.6367187649011612, "reward_std": 0.17605004459619522, "rewards/accuracy_reward/mean": 0.14732143096625805, "rewards/accuracy_reward/std": 0.3498156741261482, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.049606312066316605, "step": 878 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 934.7835235595703, "completions/mean_terminated_length": 809.6689453125, "completions/min_length": 470.75, "completions/min_terminated_length": 470.75, "epoch": 0.26256440893137184, "grad_norm": 0.37923404574394226, "kl": 1.0322265625, "learning_rate": 1.8204014435255136e-05, "loss": 0.0379, "num_tokens": 445531887.0, "reward": 0.6796875298023224, "reward_std": 0.14870525896549225, "rewards/accuracy_reward/mean": 0.1897321455180645, "rewards/accuracy_reward/std": 0.38661808520555496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04863459337502718, "step": 879 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 961.5000457763672, "completions/mean_terminated_length": 872.1953887939453, "completions/min_length": 586.75, "completions/min_terminated_length": 586.75, "epoch": 0.2628631170188933, "grad_norm": 0.3930589258670807, "kl": 0.955078125, "learning_rate": 1.8197772316385035e-05, "loss": 0.0488, "num_tokens": 446039295.0, "reward": 0.6283482313156128, "reward_std": 0.1622895672917366, "rewards/accuracy_reward/mean": 0.14099702145904303, "rewards/accuracy_reward/std": 0.3253864496946335, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 880 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 936.6986999511719, "completions/mean_terminated_length": 838.2021179199219, "completions/min_length": 522.25, "completions/min_terminated_length": 522.25, "epoch": 0.2631618251064148, "grad_norm": 0.17211687564849854, "kl": 0.884765625, "learning_rate": 1.819152044288992e-05, "loss": 0.0432, "num_tokens": 446539144.0, "reward": 0.6863839477300644, "reward_std": 0.09335624426603317, "rewards/accuracy_reward/mean": 0.19196428637951612, "rewards/accuracy_reward/std": 0.35986708849668503, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 881 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 957.6138763427734, "completions/mean_terminated_length": 812.0630493164062, "completions/min_length": 565.25, "completions/min_terminated_length": 565.25, "epoch": 0.26346053319393625, "grad_norm": 0.2838596701622009, "kl": 1.017578125, "learning_rate": 1.8185258822208968e-05, "loss": 0.0435, "num_tokens": 447038139.0, "reward": 0.5775669813156128, "reward_std": 0.09410214726813138, "rewards/accuracy_reward/mean": 0.0848214291036129, "rewards/accuracy_reward/std": 0.23521818220615387, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04000696027651429, "step": 882 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 918.1071929931641, "completions/mean_terminated_length": 821.1798400878906, "completions/min_length": 517.5, "completions/min_terminated_length": 517.5, "epoch": 0.26375924128145767, "grad_norm": 0.5853899717330933, "kl": 1.912109375, "learning_rate": 1.8178987461792955e-05, "loss": 0.0827, "num_tokens": 447516555.0, "reward": 0.5641741305589676, "reward_std": 0.14566711708903313, "rewards/accuracy_reward/mean": 0.07366071315482259, "rewards/accuracy_reward/std": 0.24320411309599876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.045961628668010235, "step": 883 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 933.7478179931641, "completions/mean_terminated_length": 825.2591247558594, "completions/min_length": 522.25, "completions/min_terminated_length": 522.25, "epoch": 0.26405794936897914, "grad_norm": 0.44935742020606995, "kl": 1.408203125, "learning_rate": 1.817270636910425e-05, "loss": 0.0633, "num_tokens": 448009130.0, "reward": 0.619419664144516, "reward_std": 0.18291859328746796, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3219386227428913, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 884 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 926.5022735595703, "completions/mean_terminated_length": 824.3045959472656, "completions/min_length": 419.25, "completions/min_terminated_length": 419.25, "epoch": 0.2643566574565006, "grad_norm": 0.2509964406490326, "kl": 1.7265625, "learning_rate": 1.816641555161679e-05, "loss": 0.0837, "num_tokens": 448494635.0, "reward": 0.7315848544239998, "reward_std": 0.16902979463338852, "rewards/accuracy_reward/mean": 0.2433035746216774, "rewards/accuracy_reward/std": 0.3440278470516205, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05178379639983177, "step": 885 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 941.7344360351562, "completions/mean_terminated_length": 859.4194488525391, "completions/min_length": 563.25, "completions/min_terminated_length": 563.25, "epoch": 0.2646553655440221, "grad_norm": 0.5804686546325684, "kl": 1.2587890625, "learning_rate": 1.81601150168161e-05, "loss": 0.0554, "num_tokens": 448984388.0, "reward": 0.7315848618745804, "reward_std": 0.15522130206227303, "rewards/accuracy_reward/mean": 0.2388392831198871, "rewards/accuracy_reward/std": 0.37452076375484467, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 886 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 951.1540679931641, "completions/mean_terminated_length": 850.1952972412109, "completions/min_length": 583.25, "completions/min_terminated_length": 583.25, "epoch": 0.26495407363154355, "grad_norm": 0.22967438399791718, "kl": 1.3759765625, "learning_rate": 1.8153804772199257e-05, "loss": 0.066, "num_tokens": 449479065.0, "reward": 0.5948660969734192, "reward_std": 0.11586167383939028, "rewards/accuracy_reward/mean": 0.10491071455180645, "rewards/accuracy_reward/std": 0.268476914614439, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04618143290281296, "step": 887 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6361607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 963.0736999511719, "completions/mean_terminated_length": 859.6719360351562, "completions/min_length": 558.0, "completions/min_terminated_length": 558.0, "epoch": 0.265252781719065, "grad_norm": 0.20013388991355896, "kl": 1.2197265625, "learning_rate": 1.8147484825274895e-05, "loss": 0.0447, "num_tokens": 449986394.0, "reward": 0.6043527275323868, "reward_std": 0.13943539559841156, "rewards/accuracy_reward/mean": 0.1138392835855484, "rewards/accuracy_reward/std": 0.3054254800081253, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574134543538094, "step": 888 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6428571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 970.4219360351562, "completions/mean_terminated_length": 873.7838897705078, "completions/min_length": 521.5, "completions/min_terminated_length": 521.5, "epoch": 0.2655514898065865, "grad_norm": 0.286338210105896, "kl": 1.2568359375, "learning_rate": 1.8141155183563195e-05, "loss": 0.0594, "num_tokens": 450493207.0, "reward": 0.5976562798023224, "reward_std": 0.15294821746647358, "rewards/accuracy_reward/mean": 0.11160714388825, "rewards/accuracy_reward/std": 0.26810422353446484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.056536297313869, "step": 889 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 962.1317291259766, "completions/mean_terminated_length": 867.1433258056641, "completions/min_length": 468.25, "completions/min_terminated_length": 468.25, "epoch": 0.26585019789410796, "grad_norm": 0.2368209809064865, "kl": 0.9375, "learning_rate": 1.8134815854595866e-05, "loss": 0.0393, "num_tokens": 451000578.0, "reward": 0.6969866454601288, "reward_std": 0.2029542401432991, "rewards/accuracy_reward/mean": 0.20982142351567745, "rewards/accuracy_reward/std": 0.39680520445108414, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.053543152287602425, "step": 890 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6450892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 966.2790679931641, "completions/mean_terminated_length": 865.1988525390625, "completions/min_length": 503.5, "completions/min_terminated_length": 503.5, "epoch": 0.26614890598162944, "grad_norm": 0.2512848377227783, "kl": 1.150390625, "learning_rate": 1.8128466845916156e-05, "loss": 0.0543, "num_tokens": 451503903.0, "reward": 0.6662946939468384, "reward_std": 0.18366591446101665, "rewards/accuracy_reward/mean": 0.19308035541325808, "rewards/accuracy_reward/std": 0.3688199818134308, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06256846059113741, "step": 891 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 943.4018249511719, "completions/mean_terminated_length": 847.9572906494141, "completions/min_length": 581.75, "completions/min_terminated_length": 581.75, "epoch": 0.2664476140691509, "grad_norm": 0.31465792655944824, "kl": 0.8662109375, "learning_rate": 1.812210816507882e-05, "loss": 0.0475, "num_tokens": 451990995.0, "reward": 0.6562500298023224, "reward_std": 0.15954172145575285, "rewards/accuracy_reward/mean": 0.16741071827709675, "rewards/accuracy_reward/std": 0.3650001883506775, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.04926478257402778, "step": 892 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 965.3437957763672, "completions/mean_terminated_length": 874.1802368164062, "completions/min_length": 557.0, "completions/min_terminated_length": 557.0, "epoch": 0.2667463221566724, "grad_norm": 0.353715717792511, "kl": 1.2529296875, "learning_rate": 1.8115739819650124e-05, "loss": 0.0612, "num_tokens": 452496381.0, "reward": 0.5742187798023224, "reward_std": 0.15558316931128502, "rewards/accuracy_reward/mean": 0.09188988036476076, "rewards/accuracy_reward/std": 0.2564760632812977, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06326735578477383, "step": 893 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 948.4710235595703, "completions/mean_terminated_length": 847.6224365234375, "completions/min_length": 545.75, "completions/min_terminated_length": 545.75, "epoch": 0.26704503024419385, "grad_norm": 0.31168997287750244, "kl": 1.1796875, "learning_rate": 1.8109361817207843e-05, "loss": 0.0521, "num_tokens": 452994448.0, "reward": 0.5535714626312256, "reward_std": 0.1284091453999281, "rewards/accuracy_reward/mean": 0.06919642840512097, "rewards/accuracy_reward/std": 0.20199326612055302, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.06033703871071339, "step": 894 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 942.3147735595703, "completions/mean_terminated_length": 823.8148956298828, "completions/min_length": 473.25, "completions/min_terminated_length": 473.25, "epoch": 0.2673437383317153, "grad_norm": 0.2731977701187134, "kl": 1.4736328125, "learning_rate": 1.8102974165341236e-05, "loss": 0.06, "num_tokens": 453484877.0, "reward": 0.5786830708384514, "reward_std": 0.1615400966256857, "rewards/accuracy_reward/mean": 0.09598214295692742, "rewards/accuracy_reward/std": 0.2589644845575094, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06278826389461756, "step": 895 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 901.8661041259766, "completions/mean_terminated_length": 812.3635711669922, "completions/min_length": 422.75, "completions/min_terminated_length": 422.75, "epoch": 0.2676424464192368, "grad_norm": 0.3192508816719055, "kl": 1.53515625, "learning_rate": 1.809657687165104e-05, "loss": 0.0726, "num_tokens": 453957345.0, "reward": 0.6199777126312256, "reward_std": 0.16709793359041214, "rewards/accuracy_reward/mean": 0.12946428544819355, "rewards/accuracy_reward/std": 0.3301222175359726, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 896 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 924.8504943847656, "completions/mean_terminated_length": 825.2466125488281, "completions/min_length": 443.25, "completions/min_terminated_length": 443.25, "epoch": 0.26794115450675826, "grad_norm": 0.3799158036708832, "kl": 2.060546875, "learning_rate": 1.8090169943749477e-05, "loss": 0.0864, "num_tokens": 454442574.0, "reward": 0.6060268133878708, "reward_std": 0.08437841292470694, "rewards/accuracy_reward/mean": 0.11830357392318547, "rewards/accuracy_reward/std": 0.2800710629671812, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05230200197547674, "step": 897 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 921.6161193847656, "completions/mean_terminated_length": 822.0244598388672, "completions/min_length": 243.25, "completions/min_terminated_length": 243.25, "epoch": 0.26823986259427973, "grad_norm": 0.46245071291923523, "kl": 1.466796875, "learning_rate": 1.8083753389260214e-05, "loss": 0.0371, "num_tokens": 454924082.0, "reward": 0.608816996216774, "reward_std": 0.17584965378046036, "rewards/accuracy_reward/mean": 0.12388393096625805, "rewards/accuracy_reward/std": 0.3160317987203598, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 898 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 897.7835235595703, "completions/mean_terminated_length": 805.9852142333984, "completions/min_length": 461.25, "completions/min_terminated_length": 461.25, "epoch": 0.2685385706818012, "grad_norm": 0.39899176359176636, "kl": 1.697265625, "learning_rate": 1.8077327215818395e-05, "loss": 0.0801, "num_tokens": 455391137.0, "reward": 0.6462053805589676, "reward_std": 0.2170945219695568, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.36384984850883484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.052645014598965645, "step": 899 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 938.7768249511719, "completions/mean_terminated_length": 843.9033355712891, "completions/min_length": 441.25, "completions/min_terminated_length": 441.25, "epoch": 0.2688372787693227, "grad_norm": 0.3292596638202667, "kl": 1.541015625, "learning_rate": 1.8070891431070597e-05, "loss": 0.0656, "num_tokens": 455882557.0, "reward": 0.5926339626312256, "reward_std": 0.15180414728820324, "rewards/accuracy_reward/mean": 0.1026785708963871, "rewards/accuracy_reward/std": 0.2830268330872059, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046074194367975, "step": 900 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 930.5513763427734, "completions/mean_terminated_length": 837.7523498535156, "completions/min_length": 507.25, "completions/min_terminated_length": 507.25, "epoch": 0.26913598685684414, "grad_norm": 0.2780855596065521, "kl": 0.8134765625, "learning_rate": 1.806444604267483e-05, "loss": 0.0363, "num_tokens": 456374628.0, "reward": 0.7070312947034836, "reward_std": 0.17386790737509727, "rewards/accuracy_reward/mean": 0.2120535746216774, "rewards/accuracy_reward/std": 0.39996373653411865, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 901 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 898.529052734375, "completions/mean_terminated_length": 812.3250732421875, "completions/min_length": 490.5, "completions/min_terminated_length": 490.5, "epoch": 0.2694346949443656, "grad_norm": 0.17424608767032623, "kl": 0.8642578125, "learning_rate": 1.8057991058300537e-05, "loss": 0.0371, "num_tokens": 456846193.0, "reward": 0.705357164144516, "reward_std": 0.233332771807909, "rewards/accuracy_reward/mean": 0.21428571455180645, "rewards/accuracy_reward/std": 0.39523912966251373, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 902 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 950.5268402099609, "completions/mean_terminated_length": 839.8440551757812, "completions/min_length": 518.25, "completions/min_terminated_length": 518.25, "epoch": 0.2697334030318871, "grad_norm": 0.587421178817749, "kl": 0.802734375, "learning_rate": 1.8051526485628582e-05, "loss": 0.0402, "num_tokens": 457342413.0, "reward": 0.6545759290456772, "reward_std": 0.14275475312024355, "rewards/accuracy_reward/mean": 0.1674107164144516, "rewards/accuracy_reward/std": 0.36862295120954514, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055120271630585194, "step": 903 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5870535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 940.7768402099609, "completions/mean_terminated_length": 822.1099700927734, "completions/min_length": 488.5, "completions/min_terminated_length": 488.5, "epoch": 0.27003211111940856, "grad_norm": 0.604601263999939, "kl": 0.7626953125, "learning_rate": 1.804505233235124e-05, "loss": 0.0469, "num_tokens": 457825897.0, "reward": 0.6333705633878708, "reward_std": 0.14741488732397556, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.3536095395684242, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05749546363949776, "step": 904 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 952.8772583007812, "completions/mean_terminated_length": 847.3306884765625, "completions/min_length": 551.5, "completions/min_terminated_length": 551.5, "epoch": 0.27033081920693003, "grad_norm": 0.230515718460083, "kl": 0.6044921875, "learning_rate": 1.8038568606172172e-05, "loss": 0.0308, "num_tokens": 458324258.0, "reward": 0.6065848469734192, "reward_std": 0.11131455423310399, "rewards/accuracy_reward/mean": 0.11383928544819355, "rewards/accuracy_reward/std": 0.26791733503341675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 905 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.609375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 964.4397735595703, "completions/mean_terminated_length": 879.6305389404297, "completions/min_length": 633.0, "completions/min_terminated_length": 633.0, "epoch": 0.2706295272944515, "grad_norm": 0.307307630777359, "kl": 0.8193359375, "learning_rate": 1.803207531480645e-05, "loss": 0.0402, "num_tokens": 458834759.0, "reward": 0.6294643133878708, "reward_std": 0.1505431141704321, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.33406205102801323, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04981599189341068, "step": 906 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 977.5781555175781, "completions/mean_terminated_length": 878.7744598388672, "completions/min_length": 618.75, "completions/min_terminated_length": 618.75, "epoch": 0.27092823538197297, "grad_norm": 0.3875752091407776, "kl": 1.20703125, "learning_rate": 1.802557246598051e-05, "loss": 0.0519, "num_tokens": 459348538.0, "reward": 0.577566996216774, "reward_std": 0.159314988180995, "rewards/accuracy_reward/mean": 0.09151785634458065, "rewards/accuracy_reward/std": 0.2889401987195015, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.05695320852100849, "step": 907 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7053571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 974.1384429931641, "completions/mean_terminated_length": 854.616455078125, "completions/min_length": 557.0, "completions/min_terminated_length": 557.0, "epoch": 0.27122694346949444, "grad_norm": 0.3173079490661621, "kl": 1.662109375, "learning_rate": 1.801906006743217e-05, "loss": 0.0729, "num_tokens": 459858760.0, "reward": 0.6562500298023224, "reward_std": 0.21599950455129147, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.3746628984808922, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.06000171788036823, "step": 908 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 947.7299499511719, "completions/mean_terminated_length": 842.72900390625, "completions/min_length": 568.5, "completions/min_terminated_length": 568.5, "epoch": 0.2715256515570159, "grad_norm": 0.4476775527000427, "kl": 1.3505859375, "learning_rate": 1.801253812691061e-05, "loss": 0.0646, "num_tokens": 460351199.0, "reward": 0.6428571790456772, "reward_std": 0.16929974034428596, "rewards/accuracy_reward/mean": 0.15178571175783873, "rewards/accuracy_reward/std": 0.3288652114570141, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 909 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 978.3661193847656, "completions/mean_terminated_length": 879.9292907714844, "completions/min_length": 625.5, "completions/min_terminated_length": 625.5, "epoch": 0.2718243596445374, "grad_norm": 0.2857663333415985, "kl": 1.45703125, "learning_rate": 1.8006006652176358e-05, "loss": 0.0668, "num_tokens": 460861843.0, "reward": 0.6305803805589676, "reward_std": 0.19696985185146332, "rewards/accuracy_reward/mean": 0.14062499813735485, "rewards/accuracy_reward/std": 0.33869557827711105, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04908842500299215, "step": 910 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 916.3482513427734, "completions/mean_terminated_length": 826.1912231445312, "completions/min_length": 538.75, "completions/min_terminated_length": 538.75, "epoch": 0.27212306773205885, "grad_norm": 0.48570069670677185, "kl": 1.48046875, "learning_rate": 1.7999465651001297e-05, "loss": 0.0628, "num_tokens": 461343935.0, "reward": 0.7232143133878708, "reward_std": 0.1666509434580803, "rewards/accuracy_reward/mean": 0.22991071082651615, "rewards/accuracy_reward/std": 0.3996233493089676, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 911 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 951.357177734375, "completions/mean_terminated_length": 831.9206695556641, "completions/min_length": 553.5, "completions/min_terminated_length": 553.5, "epoch": 0.2724217758195803, "grad_norm": 0.2410266101360321, "kl": 1.828125, "learning_rate": 1.7992915131168642e-05, "loss": 0.0833, "num_tokens": 461844671.0, "reward": 0.636160746216774, "reward_std": 0.16348514333367348, "rewards/accuracy_reward/mean": 0.14955356996506453, "rewards/accuracy_reward/std": 0.3320968635380268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05536533612757921, "step": 912 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 949.5335083007812, "completions/mean_terminated_length": 829.1378479003906, "completions/min_length": 462.75, "completions/min_terminated_length": 462.75, "epoch": 0.2727204839071018, "grad_norm": 0.1842133104801178, "kl": 1.2080078125, "learning_rate": 1.798635510047293e-05, "loss": 0.0628, "num_tokens": 462335342.0, "reward": 0.601004496216774, "reward_std": 0.16063746623694897, "rewards/accuracy_reward/mean": 0.10937499930150807, "rewards/accuracy_reward/std": 0.2857654672116041, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 913 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6473214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 966.5982513427734, "completions/mean_terminated_length": 861.0382232666016, "completions/min_length": 564.75, "completions/min_terminated_length": 564.75, "epoch": 0.27301919199462327, "grad_norm": 0.2920854091644287, "kl": 1.0517578125, "learning_rate": 1.797978556672002e-05, "loss": 0.0391, "num_tokens": 462840218.0, "reward": 0.5954241156578064, "reward_std": 0.14525574259459972, "rewards/accuracy_reward/mean": 0.10788690438494086, "rewards/accuracy_reward/std": 0.29457828029990196, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052634578198194504, "step": 914 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5892857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 958.7746124267578, "completions/mean_terminated_length": 865.141357421875, "completions/min_length": 595.5, "completions/min_terminated_length": 595.5, "epoch": 0.27331790008214474, "grad_norm": 0.27507224678993225, "kl": 0.73388671875, "learning_rate": 1.797320653772707e-05, "loss": 0.0365, "num_tokens": 463341701.0, "reward": 0.5954241305589676, "reward_std": 0.12114390917122364, "rewards/accuracy_reward/mean": 0.10267857136204839, "rewards/accuracy_reward/std": 0.24374326691031456, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03870266629382968, "step": 915 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6428571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 960.560302734375, "completions/mean_terminated_length": 849.1291046142578, "completions/min_length": 513.0, "completions/min_terminated_length": 513.0, "epoch": 0.2736166081696662, "grad_norm": 0.40653783082962036, "kl": 0.9619140625, "learning_rate": 1.7966618021322558e-05, "loss": 0.0468, "num_tokens": 463844384.0, "reward": 0.691964328289032, "reward_std": 0.19278142973780632, "rewards/accuracy_reward/mean": 0.2031250037252903, "rewards/accuracy_reward/std": 0.38148288056254387, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05146361794322729, "step": 916 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6517857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 962.216552734375, "completions/mean_terminated_length": 845.2412261962891, "completions/min_length": 578.0, "completions/min_terminated_length": 578.0, "epoch": 0.2739153162571877, "grad_norm": 0.1892499476671219, "kl": 0.5966796875, "learning_rate": 1.796002002534622e-05, "loss": 0.0252, "num_tokens": 464342753.0, "reward": 0.5708705633878708, "reward_std": 0.11401853710412979, "rewards/accuracy_reward/mean": 0.07589285774156451, "rewards/accuracy_reward/std": 0.21017154306173325, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.030101283453404903, "step": 917 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 957.9754943847656, "completions/mean_terminated_length": 850.1994171142578, "completions/min_length": 610.25, "completions/min_terminated_length": 610.25, "epoch": 0.27421402434470915, "grad_norm": 0.3619150221347809, "kl": 0.7900390625, "learning_rate": 1.79534125576491e-05, "loss": 0.0365, "num_tokens": 464843270.0, "reward": 0.6969866454601288, "reward_std": 0.18390167690813541, "rewards/accuracy_reward/mean": 0.20535714412108064, "rewards/accuracy_reward/std": 0.35898107662796974, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 918 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 922.2544860839844, "completions/mean_terminated_length": 816.5187835693359, "completions/min_length": 548.75, "completions/min_terminated_length": 548.75, "epoch": 0.2745127324322306, "grad_norm": 0.6141219139099121, "kl": 0.8408203125, "learning_rate": 1.79467956260935e-05, "loss": 0.047, "num_tokens": 465326424.0, "reward": 0.6707589477300644, "reward_std": 0.13913151063024998, "rewards/accuracy_reward/mean": 0.18303571455180645, "rewards/accuracy_reward/std": 0.36587922647595406, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 919 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6607142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 966.7745971679688, "completions/mean_terminated_length": 863.3295593261719, "completions/min_length": 518.5, "completions/min_terminated_length": 518.5, "epoch": 0.2748114405197521, "grad_norm": 0.1741885095834732, "kl": 1.1787109375, "learning_rate": 1.794016923855298e-05, "loss": 0.0531, "num_tokens": 465827747.0, "reward": 0.6841518133878708, "reward_std": 0.1772167645394802, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.37488916516304016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 920 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 945.3482666015625, "completions/mean_terminated_length": 852.2925262451172, "completions/min_length": 514.25, "completions/min_terminated_length": 514.25, "epoch": 0.27511014860727356, "grad_norm": 0.24646678566932678, "kl": 1.5478515625, "learning_rate": 1.7933533402912354e-05, "loss": 0.0752, "num_tokens": 466331183.0, "reward": 0.5613839626312256, "reward_std": 0.14497482776641846, "rewards/accuracy_reward/mean": 0.07366071501746774, "rewards/accuracy_reward/std": 0.237026609480381, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05409308057278395, "step": 921 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7321428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 976.4576416015625, "completions/mean_terminated_length": 856.9897155761719, "completions/min_length": 573.0, "completions/min_terminated_length": 573.0, "epoch": 0.27540885669479503, "grad_norm": 0.458163857460022, "kl": 1.595703125, "learning_rate": 1.7926888127067685e-05, "loss": 0.0624, "num_tokens": 466846268.0, "reward": 0.5362723469734192, "reward_std": 0.10347746312618256, "rewards/accuracy_reward/mean": 0.04241071501746774, "rewards/accuracy_reward/std": 0.18018654361367226, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 922 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 919.0580749511719, "completions/mean_terminated_length": 817.9951782226562, "completions/min_length": 526.5, "completions/min_terminated_length": 526.5, "epoch": 0.2757075647823165, "grad_norm": 0.427726149559021, "kl": 1.5546875, "learning_rate": 1.7920233418926262e-05, "loss": 0.0739, "num_tokens": 467326694.0, "reward": 0.7109375298023224, "reward_std": 0.14947001077234745, "rewards/accuracy_reward/mean": 0.21874999348074198, "rewards/accuracy_reward/std": 0.3668266162276268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 923 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 955.1183624267578, "completions/mean_terminated_length": 832.3754119873047, "completions/min_length": 423.75, "completions/min_terminated_length": 423.75, "epoch": 0.276006272869838, "grad_norm": 0.36066320538520813, "kl": 1.650390625, "learning_rate": 1.7913569286406606e-05, "loss": 0.0701, "num_tokens": 467826123.0, "reward": 0.628348246216774, "reward_std": 0.11658243648707867, "rewards/accuracy_reward/mean": 0.13616071362048388, "rewards/accuracy_reward/std": 0.3159002847969532, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 924 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 929.2545013427734, "completions/mean_terminated_length": 822.0569458007812, "completions/min_length": 482.5, "completions/min_terminated_length": 482.5, "epoch": 0.27630498095735945, "grad_norm": 0.20369912683963776, "kl": 1.68359375, "learning_rate": 1.7906895737438437e-05, "loss": 0.0878, "num_tokens": 468320397.0, "reward": 0.618303582072258, "reward_std": 0.14243203774094582, "rewards/accuracy_reward/mean": 0.12946428591385484, "rewards/accuracy_reward/std": 0.3115801326930523, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392761349678, "rewards/tag_count_reward/std": 0.04965366888791323, "step": 925 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 955.9710235595703, "completions/mean_terminated_length": 864.4969177246094, "completions/min_length": 476.75, "completions/min_terminated_length": 476.75, "epoch": 0.27660368904488086, "grad_norm": 0.19192492961883545, "kl": 1.5048828125, "learning_rate": 1.790021277996269e-05, "loss": 0.0699, "num_tokens": 468820768.0, "reward": 0.632254496216774, "reward_std": 0.16899079456925392, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.3408683277666569, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05430487543344498, "step": 926 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 936.0290679931641, "completions/mean_terminated_length": 828.1819152832031, "completions/min_length": 502.75, "completions/min_terminated_length": 502.75, "epoch": 0.27690239713240233, "grad_norm": 0.32823944091796875, "kl": 1.5859375, "learning_rate": 1.78935204219315e-05, "loss": 0.066, "num_tokens": 469311533.0, "reward": 0.5870535969734192, "reward_std": 0.14961812761612236, "rewards/accuracy_reward/mean": 0.10714285727590322, "rewards/accuracy_reward/std": 0.24841826409101486, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.04890321707352996, "step": 927 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 920.8192443847656, "completions/mean_terminated_length": 840.7904205322266, "completions/min_length": 476.25, "completions/min_terminated_length": 476.25, "epoch": 0.2772011052199238, "grad_norm": 0.19775640964508057, "kl": 0.94921875, "learning_rate": 1.7886818671308182e-05, "loss": 0.0526, "num_tokens": 469800748.0, "reward": 0.7360491305589676, "reward_std": 0.23253305070102215, "rewards/accuracy_reward/mean": 0.2410714216530323, "rewards/accuracy_reward/std": 0.40012308955192566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.030409175902605057, "step": 928 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 951.5670318603516, "completions/mean_terminated_length": 837.5146026611328, "completions/min_length": 518.0, "completions/min_terminated_length": 518.0, "epoch": 0.2774998133074453, "grad_norm": 0.22700808942317963, "kl": 1.08203125, "learning_rate": 1.788010753606722e-05, "loss": 0.0502, "num_tokens": 470299514.0, "reward": 0.5736607313156128, "reward_std": 0.14788231626152992, "rewards/accuracy_reward/mean": 0.08035714132711291, "rewards/accuracy_reward/std": 0.2523204945027828, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.039929782040417194, "step": 929 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 926.9442443847656, "completions/mean_terminated_length": 836.8274993896484, "completions/min_length": 472.5, "completions/min_terminated_length": 472.5, "epoch": 0.27779852139496675, "grad_norm": 0.2856167256832123, "kl": 0.81787109375, "learning_rate": 1.7873387024194278e-05, "loss": 0.0271, "num_tokens": 470784353.0, "reward": 0.6350446790456772, "reward_std": 0.13484717393293977, "rewards/accuracy_reward/mean": 0.14657738246023655, "rewards/accuracy_reward/std": 0.3412161022424698, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 930 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 934.4420166015625, "completions/mean_terminated_length": 834.9931488037109, "completions/min_length": 458.0, "completions/min_terminated_length": 458.0, "epoch": 0.2780972294824882, "grad_norm": 0.3759937584400177, "kl": 0.8671875, "learning_rate": 1.786665714368617e-05, "loss": 0.0415, "num_tokens": 471280439.0, "reward": 0.6512277126312256, "reward_std": 0.15977534465491772, "rewards/accuracy_reward/mean": 0.16071428824216127, "rewards/accuracy_reward/std": 0.31885068863630295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.038956642150878906, "step": 931 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 946.1161193847656, "completions/mean_terminated_length": 825.1296844482422, "completions/min_length": 504.25, "completions/min_terminated_length": 504.25, "epoch": 0.2783959375700097, "grad_norm": 0.4836122393608093, "kl": 0.6953125, "learning_rate": 1.785991790255086e-05, "loss": 0.0253, "num_tokens": 471782507.0, "reward": 0.5251116305589676, "reward_std": 0.09680849313735962, "rewards/accuracy_reward/mean": 0.03571428614668548, "rewards/accuracy_reward/std": 0.17681734822690487, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 932 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 943.5960388183594, "completions/mean_terminated_length": 813.9254608154297, "completions/min_length": 507.5, "completions/min_terminated_length": 507.5, "epoch": 0.27869464565753116, "grad_norm": 0.4375273585319519, "kl": 0.84765625, "learning_rate": 1.785316930880745e-05, "loss": 0.0417, "num_tokens": 472277558.0, "reward": 0.6852678805589676, "reward_std": 0.15244397148489952, "rewards/accuracy_reward/mean": 0.19642856949940324, "rewards/accuracy_reward/std": 0.35317564383149147, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 933 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 961.0067443847656, "completions/mean_terminated_length": 865.4599456787109, "completions/min_length": 556.5, "completions/min_terminated_length": 556.5, "epoch": 0.27899335374505263, "grad_norm": 0.2689051330089569, "kl": 0.77197265625, "learning_rate": 1.784641137048617e-05, "loss": 0.0374, "num_tokens": 472778537.0, "reward": 0.6077009215950966, "reward_std": 0.053233304526656866, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.24683281034231186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03667048690840602, "step": 934 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 936.6629943847656, "completions/mean_terminated_length": 829.1224822998047, "completions/min_length": 514.0, "completions/min_terminated_length": 514.0, "epoch": 0.2792920618325741, "grad_norm": 0.38000163435935974, "kl": 1.2314453125, "learning_rate": 1.7839644095628368e-05, "loss": 0.0692, "num_tokens": 473266370.0, "reward": 0.718191996216774, "reward_std": 0.20983733236789703, "rewards/accuracy_reward/mean": 0.2299107126891613, "rewards/accuracy_reward/std": 0.4145897477865219, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05209200643002987, "step": 935 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 975.3750457763672, "completions/mean_terminated_length": 884.1903533935547, "completions/min_length": 627.75, "completions/min_terminated_length": 627.75, "epoch": 0.27959076992009557, "grad_norm": 0.20248806476593018, "kl": 1.3544921875, "learning_rate": 1.7832867492286506e-05, "loss": 0.0616, "num_tokens": 473781242.0, "reward": 0.5954241380095482, "reward_std": 0.10881709028035402, "rewards/accuracy_reward/mean": 0.10714286006987095, "rewards/accuracy_reward/std": 0.2570314034819603, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.0502365012653172, "step": 936 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 950.5045013427734, "completions/mean_terminated_length": 858.0043029785156, "completions/min_length": 615.75, "completions/min_terminated_length": 615.75, "epoch": 0.27988947800761704, "grad_norm": 0.23611773550510406, "kl": 1.9921875, "learning_rate": 1.782608156852414e-05, "loss": 0.0937, "num_tokens": 474273500.0, "reward": 0.6300223469734192, "reward_std": 0.19408072531223297, "rewards/accuracy_reward/mean": 0.14508928637951612, "rewards/accuracy_reward/std": 0.3293149210512638, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05972688551992178, "step": 937 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5870535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 953.982177734375, "completions/mean_terminated_length": 855.5237579345703, "completions/min_length": 569.75, "completions/min_terminated_length": 569.75, "epoch": 0.2801881860951385, "grad_norm": 0.69654780626297, "kl": 1.873046875, "learning_rate": 1.7819286332415924e-05, "loss": 0.0772, "num_tokens": 474773108.0, "reward": 0.6495535969734192, "reward_std": 0.14612447284162045, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.3511238433420658, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.043861324433237314, "step": 938 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 943.2455749511719, "completions/mean_terminated_length": 835.4818420410156, "completions/min_length": 518.75, "completions/min_terminated_length": 518.75, "epoch": 0.28048689418266, "grad_norm": 0.3804914057254791, "kl": 1.8271484375, "learning_rate": 1.7812481792047587e-05, "loss": 0.0942, "num_tokens": 475266114.0, "reward": 0.632254496216774, "reward_std": 0.13954205252230167, "rewards/accuracy_reward/mean": 0.14285714272409678, "rewards/accuracy_reward/std": 0.33287595957517624, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04848270770162344, "step": 939 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 937.7031555175781, "completions/mean_terminated_length": 842.7028045654297, "completions/min_length": 551.75, "completions/min_terminated_length": 551.75, "epoch": 0.28078560227018146, "grad_norm": 0.5596436262130737, "kl": 1.921875, "learning_rate": 1.780566795551593e-05, "loss": 0.0896, "num_tokens": 475765757.0, "reward": 0.6294643133878708, "reward_std": 0.16790231317281723, "rewards/accuracy_reward/mean": 0.14062499720603228, "rewards/accuracy_reward/std": 0.32559332251548767, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05030489154160023, "step": 940 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 904.7098541259766, "completions/mean_terminated_length": 770.5297698974609, "completions/min_length": 441.75, "completions/min_terminated_length": 441.75, "epoch": 0.2810843103577029, "grad_norm": 0.2186005860567093, "kl": 1.9140625, "learning_rate": 1.7798844830928818e-05, "loss": 0.0988, "num_tokens": 476249099.0, "reward": 0.6869419887661934, "reward_std": 0.10528728924691677, "rewards/accuracy_reward/mean": 0.2008928619325161, "rewards/accuracy_reward/std": 0.32289572805166245, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05566776916384697, "step": 941 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 936.9888916015625, "completions/mean_terminated_length": 839.8581390380859, "completions/min_length": 531.75, "completions/min_terminated_length": 531.75, "epoch": 0.2813830184452244, "grad_norm": 0.19556844234466553, "kl": 1.283203125, "learning_rate": 1.779201242640517e-05, "loss": 0.0606, "num_tokens": 476742918.0, "reward": 0.642857164144516, "reward_std": 0.17353252321481705, "rewards/accuracy_reward/mean": 0.15178570989519358, "rewards/accuracy_reward/std": 0.340823233127594, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 942 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 939.0067291259766, "completions/mean_terminated_length": 822.5211486816406, "completions/min_length": 473.25, "completions/min_terminated_length": 473.25, "epoch": 0.28168172653274587, "grad_norm": 0.5526602268218994, "kl": 1.2421875, "learning_rate": 1.7785170750074937e-05, "loss": 0.0622, "num_tokens": 477231545.0, "reward": 0.6473214328289032, "reward_std": 0.15045851282775402, "rewards/accuracy_reward/mean": 0.1607142873108387, "rewards/accuracy_reward/std": 0.3686209097504616, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.055819165892899036, "step": 943 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5892857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 947.6228179931641, "completions/mean_terminated_length": 841.6814575195312, "completions/min_length": 528.75, "completions/min_terminated_length": 528.75, "epoch": 0.28198043462026734, "grad_norm": 0.4057673513889313, "kl": 1.1845703125, "learning_rate": 1.7778319810079113e-05, "loss": 0.0669, "num_tokens": 477729120.0, "reward": 0.580357164144516, "reward_std": 0.11383754387497902, "rewards/accuracy_reward/mean": 0.09151785844005644, "rewards/accuracy_reward/std": 0.25093925558030605, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 944 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 919.9978179931641, "completions/mean_terminated_length": 824.5468902587891, "completions/min_length": 575.75, "completions/min_terminated_length": 575.75, "epoch": 0.2822791427077888, "grad_norm": 0.23165249824523926, "kl": 1.458984375, "learning_rate": 1.777145961456971e-05, "loss": 0.0709, "num_tokens": 478228543.0, "reward": 0.683035746216774, "reward_std": 0.17606096528470516, "rewards/accuracy_reward/mean": 0.1964285708963871, "rewards/accuracy_reward/std": 0.3971863463521004, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05548384506255388, "step": 945 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 930.2835235595703, "completions/mean_terminated_length": 807.3977508544922, "completions/min_length": 498.75, "completions/min_terminated_length": 498.75, "epoch": 0.2825778507953103, "grad_norm": 0.18926286697387695, "kl": 1.4912109375, "learning_rate": 1.776459017170976e-05, "loss": 0.0671, "num_tokens": 478720126.0, "reward": 0.5982142984867096, "reward_std": 0.14050262048840523, "rewards/accuracy_reward/mean": 0.11160714458674192, "rewards/accuracy_reward/std": 0.30926231294870377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05451487097889185, "step": 946 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 945.0067291259766, "completions/mean_terminated_length": 839.5152740478516, "completions/min_length": 558.0, "completions/min_terminated_length": 558.0, "epoch": 0.28287655888283175, "grad_norm": 0.2641962468624115, "kl": 1.4912109375, "learning_rate": 1.7757711489673285e-05, "loss": 0.0781, "num_tokens": 479217233.0, "reward": 0.6049107238650322, "reward_std": 0.1661775391548872, "rewards/accuracy_reward/mean": 0.1160714253783226, "rewards/accuracy_reward/std": 0.26106302812695503, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050612835213541985, "step": 947 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 949.8571929931641, "completions/mean_terminated_length": 857.1341247558594, "completions/min_length": 540.0, "completions/min_terminated_length": 540.0, "epoch": 0.2831752669703532, "grad_norm": 0.20924903452396393, "kl": 1.89453125, "learning_rate": 1.7750823576645315e-05, "loss": 0.088, "num_tokens": 479720225.0, "reward": 0.6361607313156128, "reward_std": 0.16593543626368046, "rewards/accuracy_reward/mean": 0.1517857159487903, "rewards/accuracy_reward/std": 0.337939128279686, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.060200960375368595, "step": 948 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5892857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 945.0268249511719, "completions/mean_terminated_length": 832.5753784179688, "completions/min_length": 436.5, "completions/min_terminated_length": 436.5, "epoch": 0.2834739750578747, "grad_norm": 0.41837143898010254, "kl": 2.45703125, "learning_rate": 1.7743926440821857e-05, "loss": 0.0989, "num_tokens": 480218445.0, "reward": 0.5446428805589676, "reward_std": 0.15260869823396206, "rewards/accuracy_reward/mean": 0.06249999930150807, "rewards/accuracy_reward/std": 0.22842309437692165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.0635862872004509, "step": 949 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 941.0870971679688, "completions/mean_terminated_length": 845.0848541259766, "completions/min_length": 506.75, "completions/min_terminated_length": 506.75, "epoch": 0.28377268314539617, "grad_norm": 0.22419671714305878, "kl": 2.359375, "learning_rate": 1.7737020090409896e-05, "loss": 0.1105, "num_tokens": 480708132.0, "reward": 0.5714285969734192, "reward_std": 0.12093880400061607, "rewards/accuracy_reward/mean": 0.09114583348855376, "rewards/accuracy_reward/std": 0.27870194613933563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.05820113513618708, "step": 950 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 909.1763763427734, "completions/mean_terminated_length": 800.1834411621094, "completions/min_length": 487.75, "completions/min_terminated_length": 487.75, "epoch": 0.28407139123291764, "grad_norm": 0.4449227452278137, "kl": 2.904296875, "learning_rate": 1.773010453362737e-05, "loss": 0.141, "num_tokens": 481198643.0, "reward": 0.6607143133878708, "reward_std": 0.1726833339780569, "rewards/accuracy_reward/mean": 0.19345237873494625, "rewards/accuracy_reward/std": 0.3807346001267433, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.479910708963871, "rewards/tag_count_reward/std": 0.06600582599639893, "step": 951 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 924.9107666015625, "completions/mean_terminated_length": 820.0816345214844, "completions/min_length": 510.0, "completions/min_terminated_length": 510.0, "epoch": 0.2843700993204391, "grad_norm": 0.22671940922737122, "kl": 2.130859375, "learning_rate": 1.772317977870319e-05, "loss": 0.0913, "num_tokens": 481694987.0, "reward": 0.5898437723517418, "reward_std": 0.144498685374856, "rewards/accuracy_reward/mean": 0.1127232126891613, "rewards/accuracy_reward/std": 0.25893091782927513, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.0591660775244236, "step": 952 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 864.904052734375, "completions/mean_terminated_length": 796.3854827880859, "completions/min_length": 412.25, "completions/min_terminated_length": 412.25, "epoch": 0.2846688074079606, "grad_norm": 0.32416898012161255, "kl": 1.466796875, "learning_rate": 1.7716245833877202e-05, "loss": 0.0711, "num_tokens": 482157872.0, "reward": 0.7421875298023224, "reward_std": 0.23972131311893463, "rewards/accuracy_reward/mean": 0.25000000186264515, "rewards/accuracy_reward/std": 0.41963230073451996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.03658695984631777, "step": 953 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 926.8996124267578, "completions/mean_terminated_length": 806.9574127197266, "completions/min_length": 409.5, "completions/min_terminated_length": 409.5, "epoch": 0.28496751549548205, "grad_norm": 0.22561588883399963, "kl": 1.583984375, "learning_rate": 1.770930270740018e-05, "loss": 0.0734, "num_tokens": 482645027.0, "reward": 0.7126116454601288, "reward_std": 0.21532143279910088, "rewards/accuracy_reward/mean": 0.2254464328289032, "rewards/accuracy_reward/std": 0.4080696627497673, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 954 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 911.1004791259766, "completions/mean_terminated_length": 810.8260650634766, "completions/min_length": 446.5, "completions/min_terminated_length": 446.5, "epoch": 0.2852662235830035, "grad_norm": 0.16749945282936096, "kl": 1.1171875, "learning_rate": 1.7702350407533845e-05, "loss": 0.0595, "num_tokens": 483129584.0, "reward": 0.7064732313156128, "reward_std": 0.239079050719738, "rewards/accuracy_reward/mean": 0.2165178544819355, "rewards/accuracy_reward/std": 0.40172672271728516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04778412822633982, "step": 955 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.75, "completions/mean_length": 926.4978179931641, "completions/mean_terminated_length": 810.9745635986328, "completions/min_length": 560.25, "completions/min_terminated_length": 560.25, "epoch": 0.285564931670525, "grad_norm": 0.4026165306568146, "kl": 0.9404296875, "learning_rate": 1.7695388942550807e-05, "loss": 0.0445, "num_tokens": 483620191.0, "reward": 0.7382812798023224, "reward_std": 0.13467523641884327, "rewards/accuracy_reward/mean": 0.2477678544819355, "rewards/accuracy_reward/std": 0.42287133634090424, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.044524834025651217, "step": 956 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 933.0134429931641, "completions/mean_terminated_length": 827.7448272705078, "completions/min_length": 494.5, "completions/min_terminated_length": 494.5, "epoch": 0.28586363975804646, "grad_norm": 0.6702507734298706, "kl": 1.2958984375, "learning_rate": 1.7688418320734596e-05, "loss": 0.078, "num_tokens": 484110709.0, "reward": 0.6149553805589676, "reward_std": 0.13700138591229916, "rewards/accuracy_reward/mean": 0.1294642868451774, "rewards/accuracy_reward/std": 0.29184162989258766, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.057969537563622, "step": 957 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 935.7522888183594, "completions/mean_terminated_length": 841.5118408203125, "completions/min_length": 524.5, "completions/min_terminated_length": 524.5, "epoch": 0.28616234784556793, "grad_norm": 0.4675786793231964, "kl": 1.580078125, "learning_rate": 1.7681438550379645e-05, "loss": 0.0765, "num_tokens": 484599318.0, "reward": 0.6456473469734192, "reward_std": 0.13296606577932835, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.3581406995654106, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 958 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 946.4553985595703, "completions/mean_terminated_length": 838.5572509765625, "completions/min_length": 519.25, "completions/min_terminated_length": 519.25, "epoch": 0.2864610559330894, "grad_norm": 0.20603542029857635, "kl": 1.654296875, "learning_rate": 1.7674449639791255e-05, "loss": 0.0776, "num_tokens": 485088770.0, "reward": 0.6277901977300644, "reward_std": 0.1297865230590105, "rewards/accuracy_reward/mean": 0.14062500302679837, "rewards/accuracy_reward/std": 0.2956658136099577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.054124184884130955, "step": 959 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 907.154052734375, "completions/mean_terminated_length": 806.4856414794922, "completions/min_length": 434.75, "completions/min_terminated_length": 434.75, "epoch": 0.2867597640206109, "grad_norm": 0.18932750821113586, "kl": 1.7109375, "learning_rate": 1.7667451597285617e-05, "loss": 0.0823, "num_tokens": 485565911.0, "reward": 0.637276828289032, "reward_std": 0.14431294053792953, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.3443731814622879, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 960 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 926.8594360351562, "completions/mean_terminated_length": 825.1159210205078, "completions/min_length": 407.25, "completions/min_terminated_length": 407.25, "epoch": 0.28705847210813235, "grad_norm": 0.3117632269859314, "kl": 2.40625, "learning_rate": 1.766044443118978e-05, "loss": 0.1089, "num_tokens": 486052696.0, "reward": 0.628348246216774, "reward_std": 0.1977909505367279, "rewards/accuracy_reward/mean": 0.14285714272409678, "rewards/accuracy_reward/std": 0.33246706426143646, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058666424825787544, "step": 961 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 895.9710235595703, "completions/mean_terminated_length": 780.7637786865234, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "epoch": 0.2873571801956538, "grad_norm": 0.6266008019447327, "kl": 2.162109375, "learning_rate": 1.765342814984166e-05, "loss": 0.0959, "num_tokens": 486520331.0, "reward": 0.7165178805589676, "reward_std": 0.18947706371545792, "rewards/accuracy_reward/mean": 0.2276785671710968, "rewards/accuracy_reward/std": 0.41401512175798416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.0499969981610775, "step": 962 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 873.4732513427734, "completions/mean_terminated_length": 766.7232818603516, "completions/min_length": 364.5, "completions/min_terminated_length": 364.5, "epoch": 0.2876558882831753, "grad_norm": 0.3736349046230316, "kl": 2.091796875, "learning_rate": 1.7646402761590006e-05, "loss": 0.1022, "num_tokens": 486982751.0, "reward": 0.6584821790456772, "reward_std": 0.115082036703825, "rewards/accuracy_reward/mean": 0.16741071734577417, "rewards/accuracy_reward/std": 0.3492085486650467, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044658167753368616, "step": 963 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 929.9576263427734, "completions/mean_terminated_length": 825.7084655761719, "completions/min_length": 398.5, "completions/min_terminated_length": 398.5, "epoch": 0.28795459637069676, "grad_norm": 0.387058287858963, "kl": 1.708984375, "learning_rate": 1.7639368274794407e-05, "loss": 0.0853, "num_tokens": 487469052.0, "reward": 0.6450893133878708, "reward_std": 0.20031366497278214, "rewards/accuracy_reward/mean": 0.15401785261929035, "rewards/accuracy_reward/std": 0.3554486557841301, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 964 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 894.2634429931641, "completions/mean_terminated_length": 793.8394622802734, "completions/min_length": 458.5, "completions/min_terminated_length": 458.5, "epoch": 0.28825330445821823, "grad_norm": 0.1684105396270752, "kl": 1.3095703125, "learning_rate": 1.7632324697825288e-05, "loss": 0.0635, "num_tokens": 487943442.0, "reward": 0.6568080633878708, "reward_std": 0.11439952743239701, "rewards/accuracy_reward/mean": 0.16517857229337096, "rewards/accuracy_reward/std": 0.3304327577352524, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043343435507267714, "step": 965 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 878.6875457763672, "completions/mean_terminated_length": 791.0414886474609, "completions/min_length": 482.5, "completions/min_terminated_length": 482.5, "epoch": 0.2885520125457397, "grad_norm": 0.21108895540237427, "kl": 0.96923828125, "learning_rate": 1.7625272039063884e-05, "loss": 0.0536, "num_tokens": 488406710.0, "reward": 0.7025670111179352, "reward_std": 0.1409088671207428, "rewards/accuracy_reward/mean": 0.2075892835855484, "rewards/accuracy_reward/std": 0.39678939431905746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.033598463982343674, "step": 966 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 910.0647735595703, "completions/mean_terminated_length": 810.9495544433594, "completions/min_length": 344.25, "completions/min_terminated_length": 344.25, "epoch": 0.28885072063326117, "grad_norm": 0.17961958050727844, "kl": 0.9658203125, "learning_rate": 1.7618210306902227e-05, "loss": 0.0395, "num_tokens": 488885683.0, "reward": 0.603794664144516, "reward_std": 0.15954136289656162, "rewards/accuracy_reward/mean": 0.11160714318975806, "rewards/accuracy_reward/std": 0.29928476363420486, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 967 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 944.7746124267578, "completions/mean_terminated_length": 847.8092041015625, "completions/min_length": 571.5, "completions/min_terminated_length": 571.5, "epoch": 0.28914942872078264, "grad_norm": 0.24567332863807678, "kl": 0.84521484375, "learning_rate": 1.7611139509743162e-05, "loss": 0.0372, "num_tokens": 489378910.0, "reward": 0.7254464477300644, "reward_std": 0.1470230733975768, "rewards/accuracy_reward/mean": 0.2321428582072258, "rewards/accuracy_reward/std": 0.4118892699480057, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03475248999893665, "step": 968 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 948.0067443847656, "completions/mean_terminated_length": 848.8431243896484, "completions/min_length": 464.25, "completions/min_terminated_length": 464.25, "epoch": 0.28944813680830406, "grad_norm": 0.22747883200645447, "kl": 0.62548828125, "learning_rate": 1.7604059656000313e-05, "loss": 0.0276, "num_tokens": 489880081.0, "reward": 0.651785746216774, "reward_std": 0.21277366764843464, "rewards/accuracy_reward/mean": 0.15848214086145163, "rewards/accuracy_reward/std": 0.3494589179754257, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 969 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 934.591552734375, "completions/mean_terminated_length": 838.6671447753906, "completions/min_length": 526.75, "completions/min_terminated_length": 526.75, "epoch": 0.28974684489582553, "grad_norm": 0.35235461592674255, "kl": 0.74560546875, "learning_rate": 1.7596970754098074e-05, "loss": 0.0305, "num_tokens": 490372186.0, "reward": 0.7209821790456772, "reward_std": 0.16306336037814617, "rewards/accuracy_reward/mean": 0.2299107164144516, "rewards/accuracy_reward/std": 0.40925282984972, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194977223873, "step": 970 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 917.0022735595703, "completions/mean_terminated_length": 814.5947113037109, "completions/min_length": 540.0, "completions/min_terminated_length": 540.0, "epoch": 0.290045552983347, "grad_norm": 0.14789274334907532, "kl": 0.55810546875, "learning_rate": 1.758987281247162e-05, "loss": 0.034, "num_tokens": 490856219.0, "reward": 0.7829241454601288, "reward_std": 0.18581044301390648, "rewards/accuracy_reward/mean": 0.299851194024086, "rewards/accuracy_reward/std": 0.45754098892211914, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 971 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 953.3147888183594, "completions/mean_terminated_length": 855.2591705322266, "completions/min_length": 557.5, "completions/min_terminated_length": 557.5, "epoch": 0.29034426107086847, "grad_norm": 0.26615458726882935, "kl": 0.62255859375, "learning_rate": 1.758276583956687e-05, "loss": 0.034, "num_tokens": 491357304.0, "reward": 0.5870535969734192, "reward_std": 0.13344820588827133, "rewards/accuracy_reward/mean": 0.09375000116415322, "rewards/accuracy_reward/std": 0.2696064990013838, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.034245037473738194, "step": 972 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 930.3281707763672, "completions/mean_terminated_length": 837.8986358642578, "completions/min_length": 497.0, "completions/min_terminated_length": 497.0, "epoch": 0.29064296915838994, "grad_norm": 0.28693363070487976, "kl": 0.95556640625, "learning_rate": 1.75756498438405e-05, "loss": 0.0532, "num_tokens": 491850811.0, "reward": 0.6261161118745804, "reward_std": 0.1637570597231388, "rewards/accuracy_reward/mean": 0.13616071408614516, "rewards/accuracy_reward/std": 0.30545350909233093, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04508764902129769, "step": 973 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 932.0670013427734, "completions/mean_terminated_length": 810.4117126464844, "completions/min_length": 541.5, "completions/min_terminated_length": 541.5, "epoch": 0.2909416772459114, "grad_norm": 0.42822086811065674, "kl": 1.0517578125, "learning_rate": 1.7568524833759906e-05, "loss": 0.0529, "num_tokens": 492345465.0, "reward": 0.6450892984867096, "reward_std": 0.167679063975811, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3578464835882187, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 974 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 918.013427734375, "completions/mean_terminated_length": 822.8187713623047, "completions/min_length": 411.75, "completions/min_terminated_length": 411.75, "epoch": 0.2912403853334329, "grad_norm": 0.17871251702308655, "kl": 0.826171875, "learning_rate": 1.7561390817803226e-05, "loss": 0.0449, "num_tokens": 492823999.0, "reward": 0.7176339775323868, "reward_std": 0.15557556971907616, "rewards/accuracy_reward/mean": 0.23437500675208867, "rewards/accuracy_reward/std": 0.3433221634477377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03161557391285896, "step": 975 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 925.2612152099609, "completions/mean_terminated_length": 821.8721466064453, "completions/min_length": 449.25, "completions/min_terminated_length": 449.25, "epoch": 0.29153909342095435, "grad_norm": 0.20256441831588745, "kl": 1.60546875, "learning_rate": 1.7554247804459317e-05, "loss": 0.0804, "num_tokens": 493301764.0, "reward": 0.600446455180645, "reward_std": 0.14710327330976725, "rewards/accuracy_reward/mean": 0.1160714291036129, "rewards/accuracy_reward/std": 0.27152180671691895, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392984867096, "rewards/tag_count_reward/std": 0.051861658692359924, "step": 976 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 943.6094360351562, "completions/mean_terminated_length": 828.2741851806641, "completions/min_length": 522.75, "completions/min_terminated_length": 522.75, "epoch": 0.2918378015084758, "grad_norm": 0.24949178099632263, "kl": 1.607421875, "learning_rate": 1.7547095802227723e-05, "loss": 0.0736, "num_tokens": 493794069.0, "reward": 0.646763414144516, "reward_std": 0.1968252081423998, "rewards/accuracy_reward/mean": 0.1607142873108387, "rewards/accuracy_reward/std": 0.3389018438756466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05687962658703327, "step": 977 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 937.4777221679688, "completions/mean_terminated_length": 818.6897277832031, "completions/min_length": 393.5, "completions/min_terminated_length": 393.5, "epoch": 0.2921365095959973, "grad_norm": 0.39721864461898804, "kl": 3.08203125, "learning_rate": 1.7539934819618696e-05, "loss": 0.1439, "num_tokens": 494291371.0, "reward": 0.568638414144516, "reward_std": 0.10809949412941933, "rewards/accuracy_reward/mean": 0.0892857126891613, "rewards/accuracy_reward/std": 0.18907561898231506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526828289032, "rewards/tag_count_reward/std": 0.06892330199480057, "step": 978 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 934.8995819091797, "completions/mean_terminated_length": 807.4687042236328, "completions/min_length": 521.75, "completions/min_terminated_length": 521.75, "epoch": 0.29243521768351877, "grad_norm": 0.7776592969894409, "kl": 2.9453125, "learning_rate": 1.7532764865153178e-05, "loss": 0.1306, "num_tokens": 494786238.0, "reward": 0.6824777126312256, "reward_std": 0.1663947980850935, "rewards/accuracy_reward/mean": 0.19642856856808066, "rewards/accuracy_reward/std": 0.32933933287858963, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.056609878316521645, "step": 979 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 927.1763763427734, "completions/mean_terminated_length": 814.7618865966797, "completions/min_length": 541.75, "completions/min_terminated_length": 541.75, "epoch": 0.29273392577104024, "grad_norm": 0.5598598718643188, "kl": 2.791015625, "learning_rate": 1.7525585947362776e-05, "loss": 0.128, "num_tokens": 495270877.0, "reward": 0.5926339626312256, "reward_std": 0.15629038214683533, "rewards/accuracy_reward/mean": 0.10491071338765323, "rewards/accuracy_reward/std": 0.26146224699914455, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 980 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 869.0826263427734, "completions/mean_terminated_length": 768.7635955810547, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "epoch": 0.2930326338585617, "grad_norm": 0.4354497194290161, "kl": 2.884765625, "learning_rate": 1.7518398074789776e-05, "loss": 0.1435, "num_tokens": 495730034.0, "reward": 0.6445312947034836, "reward_std": 0.1626318022608757, "rewards/accuracy_reward/mean": 0.1618303582072258, "rewards/accuracy_reward/std": 0.33935531228780746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.057892187498509884, "step": 981 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 945.091552734375, "completions/mean_terminated_length": 858.2557220458984, "completions/min_length": 557.5, "completions/min_terminated_length": 557.5, "epoch": 0.2933313419460832, "grad_norm": 0.5728068947792053, "kl": 2.744140625, "learning_rate": 1.7511201255987104e-05, "loss": 0.1166, "num_tokens": 496226363.0, "reward": 0.5686384066939354, "reward_std": 0.14439806155860424, "rewards/accuracy_reward/mean": 0.08482143003493547, "rewards/accuracy_reward/std": 0.23039532452821732, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.060420761816203594, "step": 982 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 912.5670013427734, "completions/mean_terminated_length": 818.2470550537109, "completions/min_length": 453.5, "completions/min_terminated_length": 453.5, "epoch": 0.29363005003360465, "grad_norm": 0.23889292776584625, "kl": 2.267578125, "learning_rate": 1.750399549951834e-05, "loss": 0.1053, "num_tokens": 496706841.0, "reward": 0.6244419813156128, "reward_std": 0.14326105080544949, "rewards/accuracy_reward/mean": 0.14471726026386023, "rewards/accuracy_reward/std": 0.3359564170241356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.056609878316521645, "step": 983 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 899.9174652099609, "completions/mean_terminated_length": 790.1768341064453, "completions/min_length": 478.5, "completions/min_terminated_length": 478.5, "epoch": 0.2939287581211261, "grad_norm": 0.291340172290802, "kl": 2.05859375, "learning_rate": 1.74967808139577e-05, "loss": 0.1102, "num_tokens": 497174788.0, "reward": 0.647879496216774, "reward_std": 0.2213776782155037, "rewards/accuracy_reward/mean": 0.1674107126891613, "rewards/accuracy_reward/std": 0.35763826593756676, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4804687574505806, "rewards/tag_count_reward/std": 0.06571244541555643, "step": 984 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571436, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 907.6585235595703, "completions/mean_terminated_length": 809.3623657226562, "completions/min_length": 500.75, "completions/min_terminated_length": 500.75, "epoch": 0.2942274662086476, "grad_norm": 0.400542676448822, "kl": 1.287109375, "learning_rate": 1.7489557207890025e-05, "loss": 0.0589, "num_tokens": 497653451.0, "reward": 0.697544664144516, "reward_std": 0.18707172945141792, "rewards/accuracy_reward/mean": 0.2098214291036129, "rewards/accuracy_reward/std": 0.3675064742565155, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 985 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 892.154052734375, "completions/mean_terminated_length": 783.7226409912109, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "epoch": 0.29452617429616906, "grad_norm": 0.49922677874565125, "kl": 1.46484375, "learning_rate": 1.748232468991076e-05, "loss": 0.0788, "num_tokens": 498119872.0, "reward": 0.6880580633878708, "reward_std": 0.18806611746549606, "rewards/accuracy_reward/mean": 0.20312500093132257, "rewards/accuracy_reward/std": 0.34777749329805374, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05888622626662254, "step": 986 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 897.7098693847656, "completions/mean_terminated_length": 797.5667266845703, "completions/min_length": 450.25, "completions/min_terminated_length": 450.25, "epoch": 0.29482488238369053, "grad_norm": 0.5798263549804688, "kl": 1.0009765625, "learning_rate": 1.747508326862597e-05, "loss": 0.0549, "num_tokens": 498593214.0, "reward": 0.659598246216774, "reward_std": 0.20993373543024063, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.37676383554935455, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.053750067949295044, "step": 987 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 923.7411041259766, "completions/mean_terminated_length": 832.5155944824219, "completions/min_length": 535.5, "completions/min_terminated_length": 535.5, "epoch": 0.295123590471212, "grad_norm": 0.8225280046463013, "kl": 1.404296875, "learning_rate": 1.7467832952652304e-05, "loss": 0.0767, "num_tokens": 499083722.0, "reward": 0.5675223469734192, "reward_std": 0.13887345418334007, "rewards/accuracy_reward/mean": 0.08482142724096775, "rewards/accuracy_reward/std": 0.251105111092329, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06159290485084057, "step": 988 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 915.0424652099609, "completions/mean_terminated_length": 821.4593658447266, "completions/min_length": 432.75, "completions/min_terminated_length": 432.75, "epoch": 0.2954222985587335, "grad_norm": 0.1864965260028839, "kl": 1.22265625, "learning_rate": 1.7460573750616996e-05, "loss": 0.0588, "num_tokens": 499562077.0, "reward": 0.6121652126312256, "reward_std": 0.11458020214922726, "rewards/accuracy_reward/mean": 0.12053571501746774, "rewards/accuracy_reward/std": 0.24199096485972404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.0434872074984014, "step": 989 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 888.6228179931641, "completions/mean_terminated_length": 794.5249176025391, "completions/min_length": 443.5, "completions/min_terminated_length": 443.5, "epoch": 0.29572100664625495, "grad_norm": 0.253269761800766, "kl": 1.5595703125, "learning_rate": 1.745330567115786e-05, "loss": 0.0632, "num_tokens": 500038692.0, "reward": 0.6060267984867096, "reward_std": 0.1606174185872078, "rewards/accuracy_reward/mean": 0.11607142817229033, "rewards/accuracy_reward/std": 0.30080799385905266, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04716797638684511, "step": 990 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 901.4286041259766, "completions/mean_terminated_length": 799.6314849853516, "completions/min_length": 501.75, "completions/min_terminated_length": 501.75, "epoch": 0.2960197147337764, "grad_norm": 0.33672866225242615, "kl": 1.94140625, "learning_rate": 1.7446028722923266e-05, "loss": 0.101, "num_tokens": 500519668.0, "reward": 0.6037946790456772, "reward_std": 0.1510547772049904, "rewards/accuracy_reward/mean": 0.11607142584398389, "rewards/accuracy_reward/std": 0.2850215993821621, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 991 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5401785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 938.0803985595703, "completions/mean_terminated_length": 838.0195770263672, "completions/min_length": 515.5, "completions/min_terminated_length": 515.5, "epoch": 0.2963184228212979, "grad_norm": 0.623382568359375, "kl": 3.29296875, "learning_rate": 1.7438742914572137e-05, "loss": 0.143, "num_tokens": 501020088.0, "reward": 0.5429687649011612, "reward_std": 0.09056353848427534, "rewards/accuracy_reward/mean": 0.05803571408614516, "rewards/accuracy_reward/std": 0.19074633717536926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05936532001942396, "step": 992 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 914.1295013427734, "completions/mean_terminated_length": 808.6457977294922, "completions/min_length": 512.0, "completions/min_terminated_length": 512.0, "epoch": 0.29661713090881936, "grad_norm": 0.42940548062324524, "kl": 2.564453125, "learning_rate": 1.7431448254773943e-05, "loss": 0.117, "num_tokens": 501500178.0, "reward": 0.5708705708384514, "reward_std": 0.13192539382725954, "rewards/accuracy_reward/mean": 0.08482143003493547, "rewards/accuracy_reward/std": 0.22278264164924622, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057406721636652946, "step": 993 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 902.5022735595703, "completions/mean_terminated_length": 772.2826690673828, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.29691583899634083, "grad_norm": 0.694438099861145, "kl": 2.87109375, "learning_rate": 1.7424144752208688e-05, "loss": 0.1352, "num_tokens": 501971667.0, "reward": 0.698660746216774, "reward_std": 0.18384046852588654, "rewards/accuracy_reward/mean": 0.21428571455180645, "rewards/accuracy_reward/std": 0.39987049996852875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.059125179424881935, "step": 994 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 922.1339721679688, "completions/mean_terminated_length": 814.1645202636719, "completions/min_length": 459.0, "completions/min_terminated_length": 459.0, "epoch": 0.2972145470838623, "grad_norm": 0.316543310880661, "kl": 2.376953125, "learning_rate": 1.7416832415566893e-05, "loss": 0.1096, "num_tokens": 502457087.0, "reward": 0.6188616454601288, "reward_std": 0.10168778570368886, "rewards/accuracy_reward/mean": 0.13169643003493547, "rewards/accuracy_reward/std": 0.25062765181064606, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.054124184884130955, "step": 995 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 913.3460235595703, "completions/mean_terminated_length": 824.8351440429688, "completions/min_length": 533.75, "completions/min_terminated_length": 533.75, "epoch": 0.2975132551713838, "grad_norm": 0.23958081007003784, "kl": 2.037109375, "learning_rate": 1.7409511253549592e-05, "loss": 0.0941, "num_tokens": 502938730.0, "reward": 0.6065848469734192, "reward_std": 0.14124880358576775, "rewards/accuracy_reward/mean": 0.12276785541325808, "rewards/accuracy_reward/std": 0.3066609166562557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06139749940484762, "step": 996 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 939.2545013427734, "completions/mean_terminated_length": 832.2530975341797, "completions/min_length": 492.25, "completions/min_terminated_length": 492.25, "epoch": 0.29781196325890524, "grad_norm": 0.5677634477615356, "kl": 1.859375, "learning_rate": 1.7402181274868323e-05, "loss": 0.0926, "num_tokens": 503431548.0, "reward": 0.7098214626312256, "reward_std": 0.18851547874510288, "rewards/accuracy_reward/mean": 0.22767857275903225, "rewards/accuracy_reward/std": 0.396553672850132, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06418562401086092, "step": 997 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 926.1786041259766, "completions/mean_terminated_length": 804.8840789794922, "completions/min_length": 392.25, "completions/min_terminated_length": 392.25, "epoch": 0.2981106713464267, "grad_norm": 0.20063051581382751, "kl": 1.34375, "learning_rate": 1.739484248824511e-05, "loss": 0.0704, "num_tokens": 503920572.0, "reward": 0.6171875298023224, "reward_std": 0.13678935170173645, "rewards/accuracy_reward/mean": 0.12723214481957257, "rewards/accuracy_reward/std": 0.2932153008878231, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046347017865628004, "step": 998 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 924.7991638183594, "completions/mean_terminated_length": 807.5711517333984, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "epoch": 0.2984093794339482, "grad_norm": 0.45232778787612915, "kl": 1.55859375, "learning_rate": 1.7387494902412462e-05, "loss": 0.0833, "num_tokens": 504400722.0, "reward": 0.6417410969734192, "reward_std": 0.13788140937685966, "rewards/accuracy_reward/mean": 0.1540178544819355, "rewards/accuracy_reward/std": 0.3000005632638931, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053606295958161354, "step": 999 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 932.3795013427734, "completions/mean_terminated_length": 823.8197174072266, "completions/min_length": 476.5, "completions/min_terminated_length": 476.5, "epoch": 0.29870808752146966, "grad_norm": 0.2029189169406891, "kl": 1.712890625, "learning_rate": 1.738013852611336e-05, "loss": 0.0915, "num_tokens": 504906924.0, "reward": 0.5513393133878708, "reward_std": 0.13841718435287476, "rewards/accuracy_reward/mean": 0.06250000093132257, "rewards/accuracy_reward/std": 0.24033106490969658, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05157411750406027, "step": 1000 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 919.9062957763672, "completions/mean_terminated_length": 802.0439758300781, "completions/min_length": 432.5, "completions/min_terminated_length": 432.5, "epoch": 0.2990067956089911, "grad_norm": 0.22812101244926453, "kl": 1.646484375, "learning_rate": 1.737277336810124e-05, "loss": 0.0724, "num_tokens": 505389122.0, "reward": 0.6612723469734192, "reward_std": 0.13793100602924824, "rewards/accuracy_reward/mean": 0.17410713923163712, "rewards/accuracy_reward/std": 0.3361457269638777, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054050604812800884, "step": 1001 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 906.0736846923828, "completions/mean_terminated_length": 795.7843627929688, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.2993055036965126, "grad_norm": 0.43907642364501953, "kl": 2.29296875, "learning_rate": 1.736539943713999e-05, "loss": 0.1186, "num_tokens": 505867491.0, "reward": 0.5747768208384514, "reward_std": 0.10045689903199673, "rewards/accuracy_reward/mean": 0.09151785867288709, "rewards/accuracy_reward/std": 0.21834203228354454, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06168138049542904, "step": 1002 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 891.6362152099609, "completions/mean_terminated_length": 780.7940063476562, "completions/min_length": 492.75, "completions/min_terminated_length": 492.75, "epoch": 0.29960421178403407, "grad_norm": 0.3067856729030609, "kl": 1.841796875, "learning_rate": 1.7358016742003937e-05, "loss": 0.0826, "num_tokens": 506337536.0, "reward": 0.5619419813156128, "reward_std": 0.1146763227880001, "rewards/accuracy_reward/mean": 0.07142857229337096, "rewards/accuracy_reward/std": 0.23676324263215065, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 1003 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 921.6049499511719, "completions/mean_terminated_length": 806.4532623291016, "completions/min_length": 463.0, "completions/min_terminated_length": 463.0, "epoch": 0.29990291987155554, "grad_norm": 0.23389458656311035, "kl": 2.41796875, "learning_rate": 1.7350625291477835e-05, "loss": 0.1166, "num_tokens": 506827583.0, "reward": 0.6757812798023224, "reward_std": 0.22664673440158367, "rewards/accuracy_reward/mean": 0.18973213667050004, "rewards/accuracy_reward/std": 0.3492523990571499, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05392276542261243, "step": 1004 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 928.0402221679688, "completions/mean_terminated_length": 825.5841064453125, "completions/min_length": 530.0, "completions/min_terminated_length": 530.0, "epoch": 0.300201627959077, "grad_norm": 0.3122665584087372, "kl": 2.6640625, "learning_rate": 1.7343225094356857e-05, "loss": 0.1148, "num_tokens": 507325665.0, "reward": 0.568080373108387, "reward_std": 0.08882391452789307, "rewards/accuracy_reward/mean": 0.0825892873108387, "rewards/accuracy_reward/std": 0.2342197224497795, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05846718233078718, "step": 1005 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 898.1183471679688, "completions/mean_terminated_length": 800.9581451416016, "completions/min_length": 442.25, "completions/min_terminated_length": 442.25, "epoch": 0.3005003360465985, "grad_norm": 0.7606204748153687, "kl": 1.755859375, "learning_rate": 1.7335816159446585e-05, "loss": 0.0775, "num_tokens": 507812870.0, "reward": 0.5468750298023224, "reward_std": 0.1048100758343935, "rewards/accuracy_reward/mean": 0.05357142933644354, "rewards/accuracy_reward/std": 0.20868407934904099, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 1006 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 878.5357666015625, "completions/mean_terminated_length": 781.4000091552734, "completions/min_length": 426.25, "completions/min_terminated_length": 426.25, "epoch": 0.30079904413411995, "grad_norm": 0.32589927315711975, "kl": 1.41796875, "learning_rate": 1.7328398495562995e-05, "loss": 0.0688, "num_tokens": 508278054.0, "reward": 0.7271205633878708, "reward_std": 0.18469692952930927, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.41378822177648544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 1007 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 914.2678985595703, "completions/mean_terminated_length": 816.4230499267578, "completions/min_length": 523.0, "completions/min_terminated_length": 523.0, "epoch": 0.3010977522216414, "grad_norm": 0.38826942443847656, "kl": 1.23974609375, "learning_rate": 1.7320972111532456e-05, "loss": 0.061, "num_tokens": 508761086.0, "reward": 0.6406250298023224, "reward_std": 0.15278072468936443, "rewards/accuracy_reward/mean": 0.1450892868451774, "rewards/accuracy_reward/std": 0.3305591270327568, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.025947765447199345, "step": 1008 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5892857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 939.2612152099609, "completions/mean_terminated_length": 829.8462219238281, "completions/min_length": 511.5, "completions/min_terminated_length": 511.5, "epoch": 0.3013964603091629, "grad_norm": 0.180135115981102, "kl": 1.1123046875, "learning_rate": 1.7313537016191706e-05, "loss": 0.053, "num_tokens": 509256099.0, "reward": 0.5937500298023224, "reward_std": 0.1374005228281021, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.2919936254620552, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1009 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 905.3906555175781, "completions/mean_terminated_length": 778.2063751220703, "completions/min_length": 428.5, "completions/min_terminated_length": 428.5, "epoch": 0.30169516839668437, "grad_norm": 0.18598507344722748, "kl": 1.2802734375, "learning_rate": 1.7306093218387853e-05, "loss": 0.0698, "num_tokens": 509735922.0, "reward": 0.697544664144516, "reward_std": 0.21070187166333199, "rewards/accuracy_reward/mean": 0.20796130783855915, "rewards/accuracy_reward/std": 0.379374198615551, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04052485013380647, "step": 1010 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 955.4643402099609, "completions/mean_terminated_length": 858.2472839355469, "completions/min_length": 510.5, "completions/min_terminated_length": 510.5, "epoch": 0.30199387648420584, "grad_norm": 0.158708855509758, "kl": 0.7734375, "learning_rate": 1.7298640726978357e-05, "loss": 0.0404, "num_tokens": 510235506.0, "reward": 0.6350446790456772, "reward_std": 0.1678389459848404, "rewards/accuracy_reward/mean": 0.14062500139698386, "rewards/accuracy_reward/std": 0.3243018165230751, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 1011 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 932.0469207763672, "completions/mean_terminated_length": 825.9602661132812, "completions/min_length": 552.5, "completions/min_terminated_length": 552.5, "epoch": 0.30229258457172725, "grad_norm": 0.15627382695674896, "kl": 0.80810546875, "learning_rate": 1.729117955083103e-05, "loss": 0.0537, "num_tokens": 510723383.0, "reward": 0.698660746216774, "reward_std": 0.1884596012532711, "rewards/accuracy_reward/mean": 0.2083333358168602, "rewards/accuracy_reward/std": 0.40264180302619934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027185317594558, "step": 1012 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 922.2053833007812, "completions/mean_terminated_length": 820.7319946289062, "completions/min_length": 457.0, "completions/min_terminated_length": 457.0, "epoch": 0.3025912926592487, "grad_norm": 0.14325004816055298, "kl": 0.8681640625, "learning_rate": 1.7283709698824004e-05, "loss": 0.0451, "num_tokens": 511209379.0, "reward": 0.7120536118745804, "reward_std": 0.11543135903775692, "rewards/accuracy_reward/mean": 0.21651785261929035, "rewards/accuracy_reward/std": 0.3782372921705246, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1013 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49107142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 928.2924499511719, "completions/mean_terminated_length": 838.8843841552734, "completions/min_length": 470.5, "completions/min_terminated_length": 470.5, "epoch": 0.3028900007467702, "grad_norm": 0.1798219531774521, "kl": 0.8544921875, "learning_rate": 1.727623117984575e-05, "loss": 0.0465, "num_tokens": 511703478.0, "reward": 0.6344866305589676, "reward_std": 0.15737680718302727, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.28380902484059334, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1014 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 921.8304138183594, "completions/mean_terminated_length": 811.5313415527344, "completions/min_length": 434.5, "completions/min_terminated_length": 434.5, "epoch": 0.30318870883429166, "grad_norm": 0.5978297591209412, "kl": 0.9755859375, "learning_rate": 1.7268744002795043e-05, "loss": 0.0599, "num_tokens": 512196714.0, "reward": 0.5965402126312256, "reward_std": 0.1398006435483694, "rewards/accuracy_reward/mean": 0.10937499906867743, "rewards/accuracy_reward/std": 0.2941730245947838, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05500977020710707, "step": 1015 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 905.1161041259766, "completions/mean_terminated_length": 793.8847045898438, "completions/min_length": 452.25, "completions/min_terminated_length": 452.25, "epoch": 0.30348741692181314, "grad_norm": 0.1996898651123047, "kl": 0.848876953125, "learning_rate": 1.726124817658096e-05, "loss": 0.0428, "num_tokens": 512673598.0, "reward": 0.6847098618745804, "reward_std": 0.14363770931959152, "rewards/accuracy_reward/mean": 0.2008928544819355, "rewards/accuracy_reward/std": 0.3945780619978905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.028717375360429287, "step": 1016 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 906.7254791259766, "completions/mean_terminated_length": 781.7428894042969, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "epoch": 0.3037861250093346, "grad_norm": 0.2189001888036728, "kl": 1.0390625, "learning_rate": 1.7253743710122877e-05, "loss": 0.0482, "num_tokens": 513148707.0, "reward": 0.6835937798023224, "reward_std": 0.12769360467791557, "rewards/accuracy_reward/mean": 0.191964291036129, "rewards/accuracy_reward/std": 0.3192112296819687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04685897007584572, "step": 1017 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 902.5469207763672, "completions/mean_terminated_length": 808.3562774658203, "completions/min_length": 442.75, "completions/min_terminated_length": 442.75, "epoch": 0.3040848330968561, "grad_norm": 0.1583649069070816, "kl": 0.82421875, "learning_rate": 1.7246230612350444e-05, "loss": 0.0512, "num_tokens": 513622952.0, "reward": 0.6529018133878708, "reward_std": 0.14722666703164577, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.3407035432755947, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.03659330680966377, "step": 1018 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 916.9509429931641, "completions/mean_terminated_length": 805.8883361816406, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "epoch": 0.30438354118437755, "grad_norm": 0.21575719118118286, "kl": 1.2763671875, "learning_rate": 1.723870889220358e-05, "loss": 0.0582, "num_tokens": 514097922.0, "reward": 0.5965402126312256, "reward_std": 0.15322733111679554, "rewards/accuracy_reward/mean": 0.10639881063252687, "rewards/accuracy_reward/std": 0.298444289714098, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1019 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 949.3259429931641, "completions/mean_terminated_length": 839.9449310302734, "completions/min_length": 516.75, "completions/min_terminated_length": 516.75, "epoch": 0.304682249271899, "grad_norm": 0.15630307793617249, "kl": 1.296875, "learning_rate": 1.7231178558632478e-05, "loss": 0.0627, "num_tokens": 514596932.0, "reward": 0.5792410969734192, "reward_std": 0.10920144664123654, "rewards/accuracy_reward/mean": 0.08705356903374195, "rewards/accuracy_reward/std": 0.23163960129022598, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.041961644776165485, "step": 1020 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 926.4553985595703, "completions/mean_terminated_length": 813.5496826171875, "completions/min_length": 473.5, "completions/min_terminated_length": 473.5, "epoch": 0.3049809573594205, "grad_norm": 0.2757773995399475, "kl": 1.3037109375, "learning_rate": 1.7223639620597556e-05, "loss": 0.0698, "num_tokens": 515081344.0, "reward": 0.672433078289032, "reward_std": 0.22286199033260345, "rewards/accuracy_reward/mean": 0.1808035704307258, "rewards/accuracy_reward/std": 0.35036174952983856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.038298643194139004, "step": 1021 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 909.1942291259766, "completions/mean_terminated_length": 800.1002044677734, "completions/min_length": 379.25, "completions/min_terminated_length": 379.25, "epoch": 0.30527966544694196, "grad_norm": 0.368883341550827, "kl": 1.181640625, "learning_rate": 1.7216092087069496e-05, "loss": 0.0643, "num_tokens": 515564247.0, "reward": 0.6835937798023224, "reward_std": 0.1816624440252781, "rewards/accuracy_reward/mean": 0.19196428451687098, "rewards/accuracy_reward/std": 0.3525773584842682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 1022 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 945.0692291259766, "completions/mean_terminated_length": 829.2322387695312, "completions/min_length": 513.25, "completions/min_terminated_length": 513.25, "epoch": 0.30557837353446343, "grad_norm": 0.3149355947971344, "kl": 1.65234375, "learning_rate": 1.720853596702919e-05, "loss": 0.0839, "num_tokens": 516053190.0, "reward": 0.6389509290456772, "reward_std": 0.18045776709914207, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.3535061702132225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.489397332072258, "rewards/tag_count_reward/std": 0.05054692644625902, "step": 1023 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 920.5692443847656, "completions/mean_terminated_length": 791.8119812011719, "completions/min_length": 376.75, "completions/min_terminated_length": 376.75, "epoch": 0.3058770816219849, "grad_norm": 0.2754010856151581, "kl": 2.51171875, "learning_rate": 1.7200971269467754e-05, "loss": 0.1075, "num_tokens": 516533941.0, "reward": 0.6250000149011612, "reward_std": 0.14584889076650143, "rewards/accuracy_reward/mean": 0.1383928544819355, "rewards/accuracy_reward/std": 0.3446198031306267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.056545503437519073, "step": 1024 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 944.0870971679688, "completions/mean_terminated_length": 850.1997833251953, "completions/min_length": 481.0, "completions/min_terminated_length": 481.0, "epoch": 0.3061757897095064, "grad_norm": 0.514840304851532, "kl": 2.65625, "learning_rate": 1.7193398003386514e-05, "loss": 0.1171, "num_tokens": 517040796.0, "reward": 0.6216518133878708, "reward_std": 0.1369810663163662, "rewards/accuracy_reward/mean": 0.13616071501746774, "rewards/accuracy_reward/std": 0.3151542954146862, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.485491082072258, "rewards/tag_count_reward/std": 0.05758010223507881, "step": 1025 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 928.6986999511719, "completions/mean_terminated_length": 808.9399719238281, "completions/min_length": 492.5, "completions/min_terminated_length": 492.5, "epoch": 0.30647449779702785, "grad_norm": 0.2758634388446808, "kl": 1.8359375, "learning_rate": 1.718581617779698e-05, "loss": 0.0923, "num_tokens": 517527845.0, "reward": 0.6729911118745804, "reward_std": 0.1696567889302969, "rewards/accuracy_reward/mean": 0.20386904664337635, "rewards/accuracy_reward/std": 0.3689340427517891, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0410674219019711, "step": 1026 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 922.7120971679688, "completions/mean_terminated_length": 798.0655975341797, "completions/min_length": 440.5, "completions/min_terminated_length": 440.5, "epoch": 0.3067732058845493, "grad_norm": 0.2577418386936188, "kl": 2.15234375, "learning_rate": 1.7178225801720865e-05, "loss": 0.0993, "num_tokens": 518011348.0, "reward": 0.6618303805589676, "reward_std": 0.1807496640831232, "rewards/accuracy_reward/mean": 0.17633928125724196, "rewards/accuracy_reward/std": 0.3363281860947609, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.058194358833134174, "step": 1027 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 921.732177734375, "completions/mean_terminated_length": 808.0113220214844, "completions/min_length": 527.5, "completions/min_terminated_length": 527.5, "epoch": 0.3070719139720708, "grad_norm": 0.2128380835056305, "kl": 2.150390625, "learning_rate": 1.717062688419004e-05, "loss": 0.1068, "num_tokens": 518492076.0, "reward": 0.585379496216774, "reward_std": 0.1417037695646286, "rewards/accuracy_reward/mean": 0.09821428405120969, "rewards/accuracy_reward/std": 0.2760295309126377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 1028 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 917.6317443847656, "completions/mean_terminated_length": 801.2382965087891, "completions/min_length": 412.75, "completions/min_terminated_length": 412.75, "epoch": 0.30737062205959226, "grad_norm": 0.36906924843788147, "kl": 2.1328125, "learning_rate": 1.7163019434246545e-05, "loss": 0.1051, "num_tokens": 518975479.0, "reward": 0.5463169813156128, "reward_std": 0.10859731957316399, "rewards/accuracy_reward/mean": 0.0602678582072258, "rewards/accuracy_reward/std": 0.20400601252913475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057406721636652946, "step": 1029 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 910.6339721679688, "completions/mean_terminated_length": 799.615966796875, "completions/min_length": 434.25, "completions/min_terminated_length": 434.25, "epoch": 0.30766933014711373, "grad_norm": 0.2524482011795044, "kl": 2.173828125, "learning_rate": 1.7155403460942574e-05, "loss": 0.1036, "num_tokens": 519459363.0, "reward": 0.5234375298023224, "reward_std": 0.10936762997880578, "rewards/accuracy_reward/mean": 0.04055059654638171, "rewards/accuracy_reward/std": 0.12653662264347076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053152467124164104, "step": 1030 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 894.8616485595703, "completions/mean_terminated_length": 793.8809967041016, "completions/min_length": 402.75, "completions/min_terminated_length": 402.75, "epoch": 0.3079680382346352, "grad_norm": 0.261766254901886, "kl": 2.267578125, "learning_rate": 1.7147778973340466e-05, "loss": 0.1111, "num_tokens": 519939061.0, "reward": 0.675223246216774, "reward_std": 0.1584785422310233, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.3877224996685982, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06633441802114248, "step": 1031 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 860.529052734375, "completions/mean_terminated_length": 749.7739105224609, "completions/min_length": 455.5, "completions/min_terminated_length": 455.5, "epoch": 0.30826674632215667, "grad_norm": 0.288425475358963, "kl": 1.837890625, "learning_rate": 1.7140145980512684e-05, "loss": 0.1099, "num_tokens": 520400562.0, "reward": 0.749441996216774, "reward_std": 0.15772567689418793, "rewards/accuracy_reward/mean": 0.261160708963871, "rewards/accuracy_reward/std": 0.4175173044204712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05363151524215937, "step": 1032 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 899.9933319091797, "completions/mean_terminated_length": 795.9615936279297, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "epoch": 0.30856545440967814, "grad_norm": 0.29848796129226685, "kl": 2.134765625, "learning_rate": 1.713250449154182e-05, "loss": 0.0984, "num_tokens": 520881599.0, "reward": 0.616629496216774, "reward_std": 0.13709132559597492, "rewards/accuracy_reward/mean": 0.1294642831198871, "rewards/accuracy_reward/std": 0.3108113259077072, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05361673329025507, "step": 1033 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 888.3281707763672, "completions/mean_terminated_length": 790.2399749755859, "completions/min_length": 461.75, "completions/min_terminated_length": 461.75, "epoch": 0.3088641624971996, "grad_norm": 0.31662997603416443, "kl": 2.060546875, "learning_rate": 1.7124854515520562e-05, "loss": 0.107, "num_tokens": 521350370.0, "reward": 0.666294664144516, "reward_std": 0.13339894451200962, "rewards/accuracy_reward/mean": 0.1785714328289032, "rewards/accuracy_reward/std": 0.38390810042619705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 1034 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 877.8772888183594, "completions/mean_terminated_length": 776.9762420654297, "completions/min_length": 453.25, "completions/min_terminated_length": 453.25, "epoch": 0.3091628705847211, "grad_norm": 0.24893100559711456, "kl": 2.119140625, "learning_rate": 1.7117196061551714e-05, "loss": 0.1032, "num_tokens": 521817419.0, "reward": 0.5669643133878708, "reward_std": 0.14740130584686995, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.22663997113704681, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05146361794322729, "step": 1035 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 914.935302734375, "completions/mean_terminated_length": 806.3834381103516, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "epoch": 0.30946157867224255, "grad_norm": 0.307664692401886, "kl": 3.265625, "learning_rate": 1.7109529138748156e-05, "loss": 0.1556, "num_tokens": 522298094.0, "reward": 0.6573660969734192, "reward_std": 0.19719445705413818, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.36946533247828484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4765624925494194, "rewards/tag_count_reward/std": 0.07279854826629162, "step": 1036 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 877.3594055175781, "completions/mean_terminated_length": 763.8365173339844, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "epoch": 0.309760286759764, "grad_norm": 0.35769936442375183, "kl": 1.974609375, "learning_rate": 1.7101853756232856e-05, "loss": 0.0842, "num_tokens": 522766079.0, "reward": 0.6540178954601288, "reward_std": 0.16704401187598705, "rewards/accuracy_reward/mean": 0.16517856810241938, "rewards/accuracy_reward/std": 0.3278271183371544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051574116572737694, "step": 1037 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 867.1272583007812, "completions/mean_terminated_length": 758.3336944580078, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.3100589948472855, "grad_norm": 0.41336503624916077, "kl": 2.04296875, "learning_rate": 1.7094169923138832e-05, "loss": 0.0893, "num_tokens": 523225528.0, "reward": 0.7310268133878708, "reward_std": 0.24705501645803452, "rewards/accuracy_reward/mean": 0.2433035671710968, "rewards/accuracy_reward/std": 0.4300271198153496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 1038 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 879.310302734375, "completions/mean_terminated_length": 755.0628967285156, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.31035770293480697, "grad_norm": 0.33324509859085083, "kl": 1.75390625, "learning_rate": 1.708647764860917e-05, "loss": 0.0939, "num_tokens": 523690691.0, "reward": 0.6194196715950966, "reward_std": 0.1460011750459671, "rewards/accuracy_reward/mean": 0.13206845126114786, "rewards/accuracy_reward/std": 0.2788919247686863, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 1039 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 866.857177734375, "completions/mean_terminated_length": 767.4644317626953, "completions/min_length": 445.25, "completions/min_terminated_length": 445.25, "epoch": 0.31065641102232844, "grad_norm": 0.1960669457912445, "kl": 1.75, "learning_rate": 1.7078776941797e-05, "loss": 0.0977, "num_tokens": 524147619.0, "reward": 0.7215401977300644, "reward_std": 0.2249719835817814, "rewards/accuracy_reward/mean": 0.2321428582072258, "rewards/accuracy_reward/std": 0.3871385119855404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050148884765803814, "step": 1040 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 903.0826263427734, "completions/mean_terminated_length": 792.4387969970703, "completions/min_length": 432.25, "completions/min_terminated_length": 432.25, "epoch": 0.3109551191098499, "grad_norm": 0.25282010436058044, "kl": 1.11328125, "learning_rate": 1.7071067811865477e-05, "loss": 0.0605, "num_tokens": 524628264.0, "reward": 0.6222098469734192, "reward_std": 0.17136471718549728, "rewards/accuracy_reward/mean": 0.12946428917348385, "rewards/accuracy_reward/std": 0.32075049355626106, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.035161727108061314, "step": 1041 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 847.982177734375, "completions/mean_terminated_length": 717.2530822753906, "completions/min_length": 378.5, "completions/min_terminated_length": 378.5, "epoch": 0.3112538271973714, "grad_norm": 0.5962215662002563, "kl": 1.94921875, "learning_rate": 1.706335026798779e-05, "loss": 0.1296, "num_tokens": 525079408.0, "reward": 0.6255580633878708, "reward_std": 0.1520607229322195, "rewards/accuracy_reward/mean": 0.13839285681024194, "rewards/accuracy_reward/std": 0.32016080245375633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 1042 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 883.6228332519531, "completions/mean_terminated_length": 779.9668731689453, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "epoch": 0.31155253528489285, "grad_norm": 0.24915751814842224, "kl": 1.791015625, "learning_rate": 1.7055624319347134e-05, "loss": 0.1056, "num_tokens": 525545543.0, "reward": 0.6266741305589676, "reward_std": 0.16251551546156406, "rewards/accuracy_reward/mean": 0.1361607164144516, "rewards/accuracy_reward/std": 0.3356616869568825, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1043 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.25, "completions/mean_length": 856.7589721679688, "completions/mean_terminated_length": 744.5767059326172, "completions/min_length": 374.5, "completions/min_terminated_length": 374.5, "epoch": 0.3118512433724143, "grad_norm": 0.2723224461078644, "kl": 1.81640625, "learning_rate": 1.7047889975136702e-05, "loss": 0.0962, "num_tokens": 525998427.0, "reward": 0.5965402126312256, "reward_std": 0.13454468292184174, "rewards/accuracy_reward/mean": 0.10491071455180645, "rewards/accuracy_reward/std": 0.22904790192842484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1044 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 895.1629791259766, "completions/mean_terminated_length": 785.3411254882812, "completions/min_length": 466.25, "completions/min_terminated_length": 466.25, "epoch": 0.3121499514599358, "grad_norm": 0.37605568766593933, "kl": 1.8134765625, "learning_rate": 1.7040147244559688e-05, "loss": 0.0822, "num_tokens": 526481188.0, "reward": 0.6579241454601288, "reward_std": 0.1739959456026554, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.36438871175050735, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04537529917433858, "step": 1045 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48437499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 901.3326416015625, "completions/mean_terminated_length": 786.8960876464844, "completions/min_length": 424.0, "completions/min_terminated_length": 424.0, "epoch": 0.31244865954745726, "grad_norm": 0.24538490176200867, "kl": 1.412109375, "learning_rate": 1.7032396136829247e-05, "loss": 0.0588, "num_tokens": 526958729.0, "reward": 0.5943080633878708, "reward_std": 0.13236709870398045, "rewards/accuracy_reward/mean": 0.1026785708963871, "rewards/accuracy_reward/std": 0.302607424557209, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1046 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 936.9777221679688, "completions/mean_terminated_length": 822.0272064208984, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.31274736763497873, "grad_norm": 0.23204699158668518, "kl": 1.0478515625, "learning_rate": 1.702463666116852e-05, "loss": 0.0481, "num_tokens": 527464719.0, "reward": 0.6601562649011612, "reward_std": 0.13329488597810268, "rewards/accuracy_reward/mean": 0.16517857182770967, "rewards/accuracy_reward/std": 0.34652822464704514, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.034184794407337904, "step": 1047 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 902.0803985595703, "completions/mean_terminated_length": 805.4032135009766, "completions/min_length": 431.0, "completions/min_terminated_length": 431.0, "epoch": 0.3130460757225002, "grad_norm": 0.581003725528717, "kl": 1.6728515625, "learning_rate": 1.7016868826810597e-05, "loss": 0.0811, "num_tokens": 527941331.0, "reward": 0.6774553805589676, "reward_std": 0.15053263865411282, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3512096293270588, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.029416739474982023, "step": 1048 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47321428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 900.4553985595703, "completions/mean_terminated_length": 785.9456634521484, "completions/min_length": 388.25, "completions/min_terminated_length": 388.25, "epoch": 0.3133447838100217, "grad_norm": 0.19180269539356232, "kl": 0.8466796875, "learning_rate": 1.700909264299851e-05, "loss": 0.0339, "num_tokens": 528413935.0, "reward": 0.6551339477300644, "reward_std": 0.14639501832425594, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.3640752211213112, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 1049 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 942.3661193847656, "completions/mean_terminated_length": 837.0333404541016, "completions/min_length": 567.0, "completions/min_terminated_length": 567.0, "epoch": 0.31364349189754315, "grad_norm": 0.27464473247528076, "kl": 0.83203125, "learning_rate": 1.7001308118985237e-05, "loss": 0.0384, "num_tokens": 528907219.0, "reward": 0.6635044813156128, "reward_std": 0.170068470062688, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.37089723348617554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1050 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 909.8616638183594, "completions/mean_terminated_length": 807.6947174072266, "completions/min_length": 536.75, "completions/min_terminated_length": 536.75, "epoch": 0.3139421999850646, "grad_norm": 0.36961376667022705, "kl": 0.8359375, "learning_rate": 1.699351526403367e-05, "loss": 0.0491, "num_tokens": 529386629.0, "reward": 0.625558078289032, "reward_std": 0.14549686014652252, "rewards/accuracy_reward/mean": 0.13764880551025271, "rewards/accuracy_reward/std": 0.3162377141416073, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1051 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 956.3437957763672, "completions/mean_terminated_length": 852.891357421875, "completions/min_length": 504.25, "completions/min_terminated_length": 504.25, "epoch": 0.3142409080725861, "grad_norm": 0.36907759308815, "kl": 0.62646484375, "learning_rate": 1.6985714087416627e-05, "loss": 0.0292, "num_tokens": 529892143.0, "reward": 0.6149553805589676, "reward_std": 0.11350853554904461, "rewards/accuracy_reward/mean": 0.1205357164144516, "rewards/accuracy_reward/std": 0.32182005420327187, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 1052 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 935.8192291259766, "completions/mean_terminated_length": 850.2319488525391, "completions/min_length": 517.5, "completions/min_terminated_length": 517.5, "epoch": 0.31453961616010756, "grad_norm": 0.13008370995521545, "kl": 0.48779296875, "learning_rate": 1.6977904598416803e-05, "loss": 0.0265, "num_tokens": 530381022.0, "reward": 0.6880580484867096, "reward_std": 0.12063055671751499, "rewards/accuracy_reward/mean": 0.18973213993012905, "rewards/accuracy_reward/std": 0.3552122451364994, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.010136391967535019, "step": 1053 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 943.5670013427734, "completions/mean_terminated_length": 809.0857849121094, "completions/min_length": 531.5, "completions/min_terminated_length": 531.5, "epoch": 0.31483832424762903, "grad_norm": 0.1368590146303177, "kl": 0.701171875, "learning_rate": 1.6970086806326814e-05, "loss": 0.0355, "num_tokens": 530882700.0, "reward": 0.6183035969734192, "reward_std": 0.13119725929573178, "rewards/accuracy_reward/mean": 0.12276785564608872, "rewards/accuracy_reward/std": 0.23427306674420834, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1054 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 894.0067443847656, "completions/mean_terminated_length": 799.9562377929688, "completions/min_length": 503.5, "completions/min_terminated_length": 503.5, "epoch": 0.31513703233515045, "grad_norm": 0.16118282079696655, "kl": 0.4091796875, "learning_rate": 1.6962260720449134e-05, "loss": 0.0195, "num_tokens": 531358735.0, "reward": 0.6160714477300644, "reward_std": 0.10533870384097099, "rewards/accuracy_reward/mean": 0.11830357555299997, "rewards/accuracy_reward/std": 0.3075314983725548, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 1055 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 923.4330749511719, "completions/mean_terminated_length": 816.1189117431641, "completions/min_length": 477.25, "completions/min_terminated_length": 477.25, "epoch": 0.3154357404226719, "grad_norm": 0.11987883597612381, "kl": 0.67138671875, "learning_rate": 1.6954426350096118e-05, "loss": 0.032, "num_tokens": 531841505.0, "reward": 0.5703125149011612, "reward_std": 0.1245713580865413, "rewards/accuracy_reward/mean": 0.07589285913854837, "rewards/accuracy_reward/std": 0.20985163748264313, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 1056 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 951.1317443847656, "completions/mean_terminated_length": 860.14990234375, "completions/min_length": 549.75, "completions/min_terminated_length": 549.75, "epoch": 0.3157344485101934, "grad_norm": 0.17146094143390656, "kl": 0.955078125, "learning_rate": 1.6946583704589973e-05, "loss": 0.0431, "num_tokens": 532336060.0, "reward": 0.5630580633878708, "reward_std": 0.1289377510547638, "rewards/accuracy_reward/mean": 0.07142857229337096, "rewards/accuracy_reward/std": 0.2433466874063015, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 1057 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6785714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 975.0045013427734, "completions/mean_terminated_length": 875.1435241699219, "completions/min_length": 570.75, "completions/min_terminated_length": 570.75, "epoch": 0.31603315659771486, "grad_norm": 0.12955611944198608, "kl": 0.88037109375, "learning_rate": 1.693873279326276e-05, "loss": 0.0414, "num_tokens": 532849822.0, "reward": 0.6093750298023224, "reward_std": 0.15709614753723145, "rewards/accuracy_reward/mean": 0.11607143003493547, "rewards/accuracy_reward/std": 0.2526152953505516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03300748532637954, "step": 1058 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 951.1116638183594, "completions/mean_terminated_length": 837.5499877929688, "completions/min_length": 564.0, "completions/min_terminated_length": 564.0, "epoch": 0.31633186468523633, "grad_norm": 0.2287784218788147, "kl": 0.8935546875, "learning_rate": 1.6930873625456362e-05, "loss": 0.0464, "num_tokens": 533351184.0, "reward": 0.6356027126312256, "reward_std": 0.09375524520874023, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.2903364971280098, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03541599866002798, "step": 1059 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 958.5201416015625, "completions/mean_terminated_length": 872.7470092773438, "completions/min_length": 542.0, "completions/min_terminated_length": 542.0, "epoch": 0.3166305727727578, "grad_norm": 0.18948087096214294, "kl": 0.701171875, "learning_rate": 1.6923006210522497e-05, "loss": 0.0383, "num_tokens": 533859657.0, "reward": 0.6763393133878708, "reward_std": 0.21254948899149895, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.380728155374527, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1060 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 925.9375457763672, "completions/mean_terminated_length": 817.8497314453125, "completions/min_length": 480.25, "completions/min_terminated_length": 480.25, "epoch": 0.31692928086027927, "grad_norm": 0.18253277242183685, "kl": 0.796875, "learning_rate": 1.6915130557822698e-05, "loss": 0.0436, "num_tokens": 534344973.0, "reward": 0.6718750149011612, "reward_std": 0.1910049505531788, "rewards/accuracy_reward/mean": 0.17857143096625805, "rewards/accuracy_reward/std": 0.3621052876114845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 1061 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 945.0067443847656, "completions/mean_terminated_length": 843.5243377685547, "completions/min_length": 520.0, "completions/min_terminated_length": 520.0, "epoch": 0.31722798894780074, "grad_norm": 0.5163261890411377, "kl": 1.3544921875, "learning_rate": 1.690724667672829e-05, "loss": 0.0655, "num_tokens": 534845120.0, "reward": 0.6160714477300644, "reward_std": 0.17658323422074318, "rewards/accuracy_reward/mean": 0.12723214458674192, "rewards/accuracy_reward/std": 0.32588184624910355, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05157411750406027, "step": 1062 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 942.0714721679688, "completions/mean_terminated_length": 824.7080993652344, "completions/min_length": 503.25, "completions/min_terminated_length": 503.25, "epoch": 0.3175266970353222, "grad_norm": 0.1952059268951416, "kl": 1.11279296875, "learning_rate": 1.6899354576620396e-05, "loss": 0.0577, "num_tokens": 535334016.0, "reward": 0.5998884215950966, "reward_std": 0.08791518141515553, "rewards/accuracy_reward/mean": 0.11011905036866665, "rewards/accuracy_reward/std": 0.18978679180145264, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 1063 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 937.7678985595703, "completions/mean_terminated_length": 850.3735961914062, "completions/min_length": 570.5, "completions/min_terminated_length": 570.5, "epoch": 0.3178254051228437, "grad_norm": 0.24486878514289856, "kl": 1.310546875, "learning_rate": 1.6891454266889924e-05, "loss": 0.0686, "num_tokens": 535835272.0, "reward": 0.5931919813156128, "reward_std": 0.15380930714309216, "rewards/accuracy_reward/mean": 0.10044642630964518, "rewards/accuracy_reward/std": 0.29379909485578537, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 1064 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 971.1629943847656, "completions/mean_terminated_length": 861.2284088134766, "completions/min_length": 596.75, "completions/min_terminated_length": 596.75, "epoch": 0.31812411321036516, "grad_norm": 0.2624979019165039, "kl": 1.90625, "learning_rate": 1.688354575693754e-05, "loss": 0.0862, "num_tokens": 536341201.0, "reward": 0.5853794813156128, "reward_std": 0.13629168132320046, "rewards/accuracy_reward/mean": 0.09821429010480642, "rewards/accuracy_reward/std": 0.22472302615642548, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05430487543344498, "step": 1065 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 907.4353179931641, "completions/mean_terminated_length": 794.166748046875, "completions/min_length": 487.5, "completions/min_terminated_length": 487.5, "epoch": 0.3184228212978866, "grad_norm": 0.37276574969291687, "kl": 2.443359375, "learning_rate": 1.6875629056173674e-05, "loss": 0.1199, "num_tokens": 536816468.0, "reward": 0.654575914144516, "reward_std": 0.1957397647202015, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3695441335439682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05381597578525543, "step": 1066 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 929.4196929931641, "completions/mean_terminated_length": 828.0347137451172, "completions/min_length": 520.5, "completions/min_terminated_length": 520.5, "epoch": 0.3187215293854081, "grad_norm": 0.3497687876224518, "kl": 2.70703125, "learning_rate": 1.6867704174018503e-05, "loss": 0.129, "num_tokens": 537306688.0, "reward": 0.5518973469734192, "reward_std": 0.12848676461726427, "rewards/accuracy_reward/mean": 0.06696428405120969, "rewards/accuracy_reward/std": 0.1992209367454052, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05834365449845791, "step": 1067 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 943.263427734375, "completions/mean_terminated_length": 843.4962463378906, "completions/min_length": 592.25, "completions/min_terminated_length": 592.25, "epoch": 0.31902023747292957, "grad_norm": 0.42890939116477966, "kl": 1.8671875, "learning_rate": 1.685977111990193e-05, "loss": 0.0875, "num_tokens": 537803062.0, "reward": 0.7449776977300644, "reward_std": 0.19716455973684788, "rewards/accuracy_reward/mean": 0.2566964216530323, "rewards/accuracy_reward/std": 0.4263238310813904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05114053003489971, "step": 1068 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5870535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 946.388427734375, "completions/mean_terminated_length": 835.9326934814453, "completions/min_length": 490.75, "completions/min_terminated_length": 490.75, "epoch": 0.31931894556045104, "grad_norm": 0.5798762440681458, "kl": 1.759765625, "learning_rate": 1.685182990326359e-05, "loss": 0.084, "num_tokens": 538305460.0, "reward": 0.731026828289032, "reward_std": 0.2739306055009365, "rewards/accuracy_reward/mean": 0.24330356903374195, "rewards/accuracy_reward/std": 0.4137546196579933, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 1069 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 936.732177734375, "completions/mean_terminated_length": 827.9893341064453, "completions/min_length": 525.75, "completions/min_terminated_length": 525.75, "epoch": 0.3196176536479725, "grad_norm": 0.21869952976703644, "kl": 1.658203125, "learning_rate": 1.6843880533552838e-05, "loss": 0.0783, "num_tokens": 538802396.0, "reward": 0.6545759290456772, "reward_std": 0.1763158906251192, "rewards/accuracy_reward/mean": 0.16964285564608872, "rewards/accuracy_reward/std": 0.3225168474018574, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.0573029974475503, "step": 1070 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 926.7879943847656, "completions/mean_terminated_length": 812.0292510986328, "completions/min_length": 549.5, "completions/min_terminated_length": 549.5, "epoch": 0.319916361735494, "grad_norm": 0.4526684880256653, "kl": 1.345703125, "learning_rate": 1.6835923020228714e-05, "loss": 0.0729, "num_tokens": 539291149.0, "reward": 0.6417410969734192, "reward_std": 0.1959330067038536, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.34883057698607445, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.0622956370934844, "step": 1071 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5892857142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 953.5312805175781, "completions/mean_terminated_length": 856.1803588867188, "completions/min_length": 576.5, "completions/min_terminated_length": 576.5, "epoch": 0.32021506982301545, "grad_norm": 0.42077332735061646, "kl": 1.4609375, "learning_rate": 1.6827957372759957e-05, "loss": 0.0759, "num_tokens": 539789675.0, "reward": 0.5864955484867096, "reward_std": 0.19800801202654839, "rewards/accuracy_reward/mean": 0.10714285727590322, "rewards/accuracy_reward/std": 0.30443868786096573, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526753783226, "rewards/tag_count_reward/std": 0.06852320395410061, "step": 1072 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 958.2277221679688, "completions/mean_terminated_length": 850.6076202392578, "completions/min_length": 584.5, "completions/min_terminated_length": 584.5, "epoch": 0.3205137779105369, "grad_norm": 0.8222906589508057, "kl": 1.2021484375, "learning_rate": 1.6819983600624986e-05, "loss": 0.0603, "num_tokens": 540288961.0, "reward": 0.5809152126312256, "reward_std": 0.14924936182796955, "rewards/accuracy_reward/mean": 0.09821428451687098, "rewards/accuracy_reward/std": 0.2585136219859123, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06312516331672668, "step": 1073 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 925.8861999511719, "completions/mean_terminated_length": 831.4052124023438, "completions/min_length": 519.5, "completions/min_terminated_length": 519.5, "epoch": 0.3208124859980584, "grad_norm": 0.6748024821281433, "kl": 2.09765625, "learning_rate": 1.6812001713311887e-05, "loss": 0.1101, "num_tokens": 540774382.0, "reward": 0.5513393059372902, "reward_std": 0.16352828592061996, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.22787199169397354, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4732142835855484, "rewards/tag_count_reward/std": 0.07735671103000641, "step": 1074 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 934.6094207763672, "completions/mean_terminated_length": 815.7372283935547, "completions/min_length": 511.25, "completions/min_terminated_length": 511.25, "epoch": 0.32111119408557987, "grad_norm": 0.6521512866020203, "kl": 2.537109375, "learning_rate": 1.6804011720318394e-05, "loss": 0.13, "num_tokens": 541261295.0, "reward": 0.6456473469734192, "reward_std": 0.16231231950223446, "rewards/accuracy_reward/mean": 0.1674107164144516, "rewards/accuracy_reward/std": 0.3623759299516678, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366082072258, "rewards/tag_count_reward/std": 0.06823405995965004, "step": 1075 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 891.0379943847656, "completions/mean_terminated_length": 793.3629150390625, "completions/min_length": 431.5, "completions/min_terminated_length": 431.5, "epoch": 0.32140990217310134, "grad_norm": 0.3113571107387543, "kl": 3.59375, "learning_rate": 1.6796013631151898e-05, "loss": 0.1815, "num_tokens": 541740512.0, "reward": 0.5703125149011612, "reward_std": 0.15259125642478466, "rewards/accuracy_reward/mean": 0.09374999906867743, "rewards/accuracy_reward/std": 0.2893896996974945, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4765625, "rewards/tag_count_reward/std": 0.07317237928509712, "step": 1076 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5401785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 916.8504943847656, "completions/mean_terminated_length": 811.8292541503906, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "epoch": 0.3217086102606228, "grad_norm": 0.9099510908126831, "kl": 5.6796875, "learning_rate": 1.678800745532942e-05, "loss": 0.2706, "num_tokens": 542235805.0, "reward": 0.5546875223517418, "reward_std": 0.14649420604109764, "rewards/accuracy_reward/mean": 0.08705357182770967, "rewards/accuracy_reward/std": 0.24169250577688217, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4676339253783226, "rewards/tag_count_reward/std": 0.08381934463977814, "step": 1077 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 871.7254791259766, "completions/mean_terminated_length": 771.2676086425781, "completions/min_length": 418.25, "completions/min_terminated_length": 418.25, "epoch": 0.3220073183481443, "grad_norm": 1.0271883010864258, "kl": 5.91015625, "learning_rate": 1.6779993202377597e-05, "loss": 0.2838, "num_tokens": 542697522.0, "reward": 0.5898437798023224, "reward_std": 0.1735545303672552, "rewards/accuracy_reward/mean": 0.12053571408614516, "rewards/accuracy_reward/std": 0.30583620071411133, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.469308041036129, "rewards/tag_count_reward/std": 0.0798784401267767, "step": 1078 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 857.9531707763672, "completions/mean_terminated_length": 752.4498138427734, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "epoch": 0.32230602643566575, "grad_norm": 0.4507637321949005, "kl": 3.734375, "learning_rate": 1.677197088183269e-05, "loss": 0.1846, "num_tokens": 543152141.0, "reward": 0.6328125298023224, "reward_std": 0.18363309279084206, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.34706058725714684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06234364025294781, "step": 1079 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 834.2969055175781, "completions/mean_terminated_length": 733.9969329833984, "completions/min_length": 325.5, "completions/min_terminated_length": 325.5, "epoch": 0.3226047345231872, "grad_norm": 0.2770647704601288, "kl": 2.748046875, "learning_rate": 1.6763940503240543e-05, "loss": 0.1379, "num_tokens": 543597522.0, "reward": 0.6121652126312256, "reward_std": 0.13044501887634397, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.27888619154691696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05655108019709587, "step": 1080 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 835.7455749511719, "completions/mean_terminated_length": 760.9711151123047, "completions/min_length": 360.25, "completions/min_terminated_length": 360.25, "epoch": 0.3229034426107087, "grad_norm": 0.3491548001766205, "kl": 2.51171875, "learning_rate": 1.6755902076156606e-05, "loss": 0.1332, "num_tokens": 544056640.0, "reward": 0.6434151977300644, "reward_std": 0.15238341689109802, "rewards/accuracy_reward/mean": 0.15624999580904841, "rewards/accuracy_reward/std": 0.3381207212805748, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05233129486441612, "step": 1081 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31026785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 844.388427734375, "completions/mean_terminated_length": 763.4772644042969, "completions/min_length": 359.75, "completions/min_terminated_length": 359.75, "epoch": 0.32320215069823016, "grad_norm": 0.2801837623119354, "kl": 1.447265625, "learning_rate": 1.6747855610145885e-05, "loss": 0.0728, "num_tokens": 544499742.0, "reward": 0.6880580633878708, "reward_std": 0.1671795528382063, "rewards/accuracy_reward/mean": 0.19642857275903225, "rewards/accuracy_reward/std": 0.3871289938688278, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 1082 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33482142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 849.8549652099609, "completions/mean_terminated_length": 768.490234375, "completions/min_length": 357.5, "completions/min_terminated_length": 357.5, "epoch": 0.32350085878575163, "grad_norm": 0.5182177424430847, "kl": 1.455078125, "learning_rate": 1.673980111478298e-05, "loss": 0.0847, "num_tokens": 544951693.0, "reward": 0.5848214626312256, "reward_std": 0.13077040389180183, "rewards/accuracy_reward/mean": 0.09375000116415322, "rewards/accuracy_reward/std": 0.25619816966354847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 1083 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40178571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 880.3013916015625, "completions/mean_terminated_length": 791.6893157958984, "completions/min_length": 442.25, "completions/min_terminated_length": 442.25, "epoch": 0.3237995668732731, "grad_norm": 0.2011082023382187, "kl": 1.0498046875, "learning_rate": 1.6731738599652017e-05, "loss": 0.0649, "num_tokens": 545418868.0, "reward": 0.7215402275323868, "reward_std": 0.1641867570579052, "rewards/accuracy_reward/mean": 0.22544642654247582, "rewards/accuracy_reward/std": 0.3636856656521559, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1084 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 885.7790679931641, "completions/mean_terminated_length": 806.3280639648438, "completions/min_length": 469.75, "completions/min_terminated_length": 469.75, "epoch": 0.3240982749607946, "grad_norm": 0.27522242069244385, "kl": 1.15234375, "learning_rate": 1.672366807434668e-05, "loss": 0.0554, "num_tokens": 545883633.0, "reward": 0.6529018133878708, "reward_std": 0.20697223767638206, "rewards/accuracy_reward/mean": 0.1584821380674839, "rewards/accuracy_reward/std": 0.35089848563075066, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1085 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 878.8147735595703, "completions/mean_terminated_length": 789.4952239990234, "completions/min_length": 495.75, "completions/min_terminated_length": 495.75, "epoch": 0.32439698304831605, "grad_norm": 0.1409914195537567, "kl": 0.7998046875, "learning_rate": 1.6715589548470187e-05, "loss": 0.0339, "num_tokens": 546364542.0, "reward": 0.731026828289032, "reward_std": 0.21463290601968765, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.42225442826747894, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 1086 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 894.7701416015625, "completions/mean_terminated_length": 814.7576904296875, "completions/min_length": 395.75, "completions/min_terminated_length": 395.75, "epoch": 0.3246956911358375, "grad_norm": 0.2600575089454651, "kl": 0.80322265625, "learning_rate": 1.6707503031635258e-05, "loss": 0.0391, "num_tokens": 546840247.0, "reward": 0.646763414144516, "reward_std": 0.1130892988294363, "rewards/accuracy_reward/mean": 0.14955357182770967, "rewards/accuracy_reward/std": 0.33767833560705185, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 1087 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 892.857177734375, "completions/mean_terminated_length": 797.4132537841797, "completions/min_length": 465.75, "completions/min_terminated_length": 465.75, "epoch": 0.324994399223359, "grad_norm": 0.15999215841293335, "kl": 0.67431640625, "learning_rate": 1.6699408533464145e-05, "loss": 0.0334, "num_tokens": 547315815.0, "reward": 0.6635044813156128, "reward_std": 0.09507536189630628, "rewards/accuracy_reward/mean": 0.1674107164144516, "rewards/accuracy_reward/std": 0.31017860025167465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 1088 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 899.5915679931641, "completions/mean_terminated_length": 792.33935546875, "completions/min_length": 360.5, "completions/min_terminated_length": 360.5, "epoch": 0.32529310731088046, "grad_norm": 0.13507744669914246, "kl": 0.87890625, "learning_rate": 1.6691306063588583e-05, "loss": 0.0402, "num_tokens": 547793296.0, "reward": 0.6852678954601288, "reward_std": 0.1401341687887907, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.370360866189003, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03992978297173977, "step": 1089 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 949.4732513427734, "completions/mean_terminated_length": 846.3888549804688, "completions/min_length": 523.5, "completions/min_terminated_length": 523.5, "epoch": 0.32559181539840193, "grad_norm": 0.21684768795967102, "kl": 0.689453125, "learning_rate": 1.6683195631649795e-05, "loss": 0.0333, "num_tokens": 548289780.0, "reward": 0.6378348469734192, "reward_std": 0.13255843706429005, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.32863863185048103, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.03309101238846779, "step": 1090 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 958.5156707763672, "completions/mean_terminated_length": 865.7695007324219, "completions/min_length": 518.25, "completions/min_terminated_length": 518.25, "epoch": 0.3258905234859234, "grad_norm": 0.16792775690555573, "kl": 0.669921875, "learning_rate": 1.6675077247298475e-05, "loss": 0.0396, "num_tokens": 548787771.0, "reward": 0.7349330633878708, "reward_std": 0.18090418842621148, "rewards/accuracy_reward/mean": 0.2388392835855484, "rewards/accuracy_reward/std": 0.31687015295028687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1091 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 932.3147583007812, "completions/mean_terminated_length": 833.3060150146484, "completions/min_length": 525.25, "completions/min_terminated_length": 525.25, "epoch": 0.32618923157344487, "grad_norm": 0.2501113712787628, "kl": 0.74853515625, "learning_rate": 1.666695092019479e-05, "loss": 0.0348, "num_tokens": 549274664.0, "reward": 0.678013414144516, "reward_std": 0.2105015330016613, "rewards/accuracy_reward/mean": 0.1808035671710968, "rewards/accuracy_reward/std": 0.3656196743249893, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.01845060009509325, "step": 1092 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 944.2455902099609, "completions/mean_terminated_length": 820.1519317626953, "completions/min_length": 525.5, "completions/min_terminated_length": 525.5, "epoch": 0.32648793966096634, "grad_norm": 0.20029354095458984, "kl": 0.529052734375, "learning_rate": 1.6658816660008344e-05, "loss": 0.0286, "num_tokens": 549770822.0, "reward": 0.602120578289032, "reward_std": 0.14211264811456203, "rewards/accuracy_reward/mean": 0.10714285913854837, "rewards/accuracy_reward/std": 0.3020046427845955, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.033598463982343674, "step": 1093 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 932.8303985595703, "completions/mean_terminated_length": 802.3775482177734, "completions/min_length": 417.75, "completions/min_terminated_length": 417.75, "epoch": 0.3267866477484878, "grad_norm": 0.1973211020231247, "kl": 0.86474609375, "learning_rate": 1.6650674476418193e-05, "loss": 0.0451, "num_tokens": 550260794.0, "reward": 0.6813616454601288, "reward_std": 0.11386625282466412, "rewards/accuracy_reward/mean": 0.1927083320915699, "rewards/accuracy_reward/std": 0.3908236622810364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616156578064, "rewards/tag_count_reward/std": 0.03183652414008975, "step": 1094 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 951.1853179931641, "completions/mean_terminated_length": 869.4385833740234, "completions/min_length": 585.5, "completions/min_terminated_length": 585.5, "epoch": 0.3270853558360093, "grad_norm": 0.3379593789577484, "kl": 0.796875, "learning_rate": 1.664252437911282e-05, "loss": 0.0455, "num_tokens": 550756797.0, "reward": 0.5820312649011612, "reward_std": 0.09619971411302686, "rewards/accuracy_reward/mean": 0.08928571501746774, "rewards/accuracy_reward/std": 0.21540624275803566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196588039398, "step": 1095 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 945.888427734375, "completions/mean_terminated_length": 822.7760009765625, "completions/min_length": 537.0, "completions/min_terminated_length": 537.0, "epoch": 0.32738406392353075, "grad_norm": 0.21045024693012238, "kl": 1.0947265625, "learning_rate": 1.6634366377790113e-05, "loss": 0.0486, "num_tokens": 551247163.0, "reward": 0.6607143133878708, "reward_std": 0.1551051065325737, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.35931258276104927, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05277476366609335, "step": 1096 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.640625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 963.3772735595703, "completions/mean_terminated_length": 858.5027923583984, "completions/min_length": 613.25, "completions/min_terminated_length": 613.25, "epoch": 0.3276827720110522, "grad_norm": 0.18472108244895935, "kl": 1.322265625, "learning_rate": 1.6626200482157378e-05, "loss": 0.0622, "num_tokens": 551752740.0, "reward": 0.6395089626312256, "reward_std": 0.15289154089987278, "rewards/accuracy_reward/mean": 0.1495535676367581, "rewards/accuracy_reward/std": 0.3184468112885952, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04666052386164665, "step": 1097 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6986607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 972.6339721679688, "completions/mean_terminated_length": 851.0587158203125, "completions/min_length": 554.0, "completions/min_terminated_length": 554.0, "epoch": 0.32798148009857364, "grad_norm": 0.16433113813400269, "kl": 1.326171875, "learning_rate": 1.6618026701931308e-05, "loss": 0.0555, "num_tokens": 552261408.0, "reward": 0.5898437649011612, "reward_std": 0.0907585141249001, "rewards/accuracy_reward/mean": 0.09821428917348385, "rewards/accuracy_reward/std": 0.2772590145468712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.043475935235619545, "step": 1098 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6986607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 972.5045013427734, "completions/mean_terminated_length": 859.0086517333984, "completions/min_length": 547.0, "completions/min_terminated_length": 547.0, "epoch": 0.3282801881860951, "grad_norm": 0.27983972430229187, "kl": 1.798828125, "learning_rate": 1.6609845046837976e-05, "loss": 0.0818, "num_tokens": 552772162.0, "reward": 0.6311384066939354, "reward_std": 0.1678405674174428, "rewards/accuracy_reward/mean": 0.1514136902987957, "rewards/accuracy_reward/std": 0.3013663962483406, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05713389813899994, "step": 1099 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7477678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 984.1696929931641, "completions/mean_terminated_length": 867.5233459472656, "completions/min_length": 589.25, "completions/min_terminated_length": 589.25, "epoch": 0.3285788962736166, "grad_norm": 0.7868248224258423, "kl": 2.384765625, "learning_rate": 1.6601655526612836e-05, "loss": 0.109, "num_tokens": 553293006.0, "reward": 0.604910746216774, "reward_std": 0.15937870927155018, "rewards/accuracy_reward/mean": 0.12499999906867743, "rewards/accuracy_reward/std": 0.314626757055521, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4799107164144516, "rewards/tag_count_reward/std": 0.067145686596632, "step": 1100 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 981.3393402099609, "completions/mean_terminated_length": 886.8535614013672, "completions/min_length": 685.0, "completions/min_terminated_length": 685.0, "epoch": 0.32887760436113805, "grad_norm": 0.32241201400756836, "kl": 2.224609375, "learning_rate": 1.659345815100069e-05, "loss": 0.0991, "num_tokens": 553801926.0, "reward": 0.603794664144516, "reward_std": 0.1433268878608942, "rewards/accuracy_reward/mean": 0.12127975886687636, "rewards/accuracy_reward/std": 0.2941870875656605, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06236921809613705, "step": 1101 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7008928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 980.591552734375, "completions/mean_terminated_length": 875.5716400146484, "completions/min_length": 620.5, "completions/min_terminated_length": 620.5, "epoch": 0.3291763124486595, "grad_norm": 0.3575092554092407, "kl": 3.0078125, "learning_rate": 1.6585252929755693e-05, "loss": 0.1304, "num_tokens": 554319583.0, "reward": 0.6043526977300644, "reward_std": 0.17939742375165224, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.2853783965110779, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.47265625, "rewards/tag_count_reward/std": 0.07766447961330414, "step": 1102 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7857142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 992.2299652099609, "completions/mean_terminated_length": 877.1153411865234, "completions/min_length": 630.25, "completions/min_terminated_length": 630.25, "epoch": 0.329475020536181, "grad_norm": 0.39065396785736084, "kl": 2.443359375, "learning_rate": 1.657703987264133e-05, "loss": 0.101, "num_tokens": 554839590.0, "reward": 0.6065848469734192, "reward_std": 0.1443085726350546, "rewards/accuracy_reward/mean": 0.1316964286379516, "rewards/accuracy_reward/std": 0.25496111810207367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883843421936, "rewards/tag_count_reward/std": 0.07046653889119625, "step": 1103 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7075892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 977.7723693847656, "completions/mean_terminated_length": 866.1147308349609, "completions/min_length": 608.5, "completions/min_terminated_length": 608.5, "epoch": 0.32977372862370247, "grad_norm": 0.27663087844848633, "kl": 2.54296875, "learning_rate": 1.6568818989430416e-05, "loss": 0.1175, "num_tokens": 555358512.0, "reward": 0.5585937798023224, "reward_std": 0.17339811101555824, "rewards/accuracy_reward/mean": 0.08258928684517741, "rewards/accuracy_reward/std": 0.26304866746068, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4760044664144516, "rewards/tag_count_reward/std": 0.07235876843333244, "step": 1104 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7142857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 975.6563110351562, "completions/mean_terminated_length": 857.4409484863281, "completions/min_length": 549.5, "completions/min_terminated_length": 549.5, "epoch": 0.33007243671122394, "grad_norm": 0.2839711904525757, "kl": 2.55078125, "learning_rate": 1.6560590289905074e-05, "loss": 0.1083, "num_tokens": 555869270.0, "reward": 0.5658482313156128, "reward_std": 0.17475899029523134, "rewards/accuracy_reward/mean": 0.08928571455180645, "rewards/accuracy_reward/std": 0.2300296388566494, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4765624925494194, "rewards/tag_count_reward/std": 0.07279855012893677, "step": 1105 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6361607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 958.7835235595703, "completions/mean_terminated_length": 842.3141021728516, "completions/min_length": 554.25, "completions/min_terminated_length": 554.25, "epoch": 0.3303711447987454, "grad_norm": 0.642437756061554, "kl": 2.62890625, "learning_rate": 1.6552353783856733e-05, "loss": 0.1261, "num_tokens": 556368917.0, "reward": 0.5887276977300644, "reward_std": 0.14578337594866753, "rewards/accuracy_reward/mean": 0.1160714291036129, "rewards/accuracy_reward/std": 0.3146461322903633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.47265625, "rewards/tag_count_reward/std": 0.07819235138595104, "step": 1106 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6607142857142856, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 955.9353179931641, "completions/mean_terminated_length": 827.2630767822266, "completions/min_length": 483.75, "completions/min_terminated_length": 483.75, "epoch": 0.3306698528862669, "grad_norm": 0.3032834231853485, "kl": 3.4375, "learning_rate": 1.6544109481086106e-05, "loss": 0.1653, "num_tokens": 556870472.0, "reward": 0.566964328289032, "reward_std": 0.18602709285914898, "rewards/accuracy_reward/mean": 0.09635416860692203, "rewards/accuracy_reward/std": 0.26326877996325493, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4709821417927742, "rewards/tag_count_reward/std": 0.078978450037539, "step": 1107 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 922.6451416015625, "completions/mean_terminated_length": 802.7613830566406, "completions/min_length": 424.5, "completions/min_terminated_length": 424.5, "epoch": 0.33096856097378835, "grad_norm": 0.5331261157989502, "kl": 2.87890625, "learning_rate": 1.6535857391403186e-05, "loss": 0.1312, "num_tokens": 557359065.0, "reward": 0.6886160969734192, "reward_std": 0.17282747477293015, "rewards/accuracy_reward/mean": 0.20982142724096775, "rewards/accuracy_reward/std": 0.40021052956581116, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.0696312440559268, "step": 1108 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 952.7723693847656, "completions/mean_terminated_length": 843.3165893554688, "completions/min_length": 505.25, "completions/min_terminated_length": 505.25, "epoch": 0.3312672690613098, "grad_norm": 0.3048416078090668, "kl": 3.046875, "learning_rate": 1.6527597524627226e-05, "loss": 0.1404, "num_tokens": 557854211.0, "reward": 0.5563616305589676, "reward_std": 0.15701443888247013, "rewards/accuracy_reward/mean": 0.0807291662786156, "rewards/accuracy_reward/std": 0.243526054546237, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366156578064, "rewards/tag_count_reward/std": 0.07043877243995667, "step": 1109 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 918.9442443847656, "completions/mean_terminated_length": 794.7108612060547, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "epoch": 0.3315659771488313, "grad_norm": 0.669957160949707, "kl": 3.0703125, "learning_rate": 1.6519329890586743e-05, "loss": 0.1413, "num_tokens": 558353098.0, "reward": 0.5915178954601288, "reward_std": 0.22334383241832256, "rewards/accuracy_reward/mean": 0.10937500093132257, "rewards/accuracy_reward/std": 0.30089009553194046, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.059903773944824934, "step": 1110 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 935.6138763427734, "completions/mean_terminated_length": 813.6683197021484, "completions/min_length": 459.75, "completions/min_terminated_length": 459.75, "epoch": 0.33186468523635276, "grad_norm": 0.22501549124717712, "kl": 2.515625, "learning_rate": 1.6511054499119493e-05, "loss": 0.1226, "num_tokens": 558842733.0, "reward": 0.5747768133878708, "reward_std": 0.1259177215397358, "rewards/accuracy_reward/mean": 0.0915178582072258, "rewards/accuracy_reward/std": 0.19367394596338272, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.05866044154390693, "step": 1111 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 915.9219207763672, "completions/mean_terminated_length": 808.1417846679688, "completions/min_length": 471.5, "completions/min_terminated_length": 471.5, "epoch": 0.33216339332387423, "grad_norm": 0.25576135516166687, "kl": 2.03515625, "learning_rate": 1.6502771360072457e-05, "loss": 0.1098, "num_tokens": 559326794.0, "reward": 0.6696428954601288, "reward_std": 0.2199811190366745, "rewards/accuracy_reward/mean": 0.18526785634458065, "rewards/accuracy_reward/std": 0.37228507548570633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05992202274501324, "step": 1112 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 903.6763916015625, "completions/mean_terminated_length": 804.2456512451172, "completions/min_length": 365.75, "completions/min_terminated_length": 365.75, "epoch": 0.3324621014113957, "grad_norm": 0.26138046383857727, "kl": 1.734375, "learning_rate": 1.6494480483301836e-05, "loss": 0.0921, "num_tokens": 559811785.0, "reward": 0.5747768133878708, "reward_std": 0.12236590683460236, "rewards/accuracy_reward/mean": 0.08705357112921774, "rewards/accuracy_reward/std": 0.2529203649610281, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 1113 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 876.3103179931641, "completions/mean_terminated_length": 774.3348693847656, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "epoch": 0.3327608094989172, "grad_norm": 0.30616992712020874, "kl": 1.2392578125, "learning_rate": 1.648618187867305e-05, "loss": 0.0735, "num_tokens": 560273764.0, "reward": 0.675223246216774, "reward_std": 0.17690913751721382, "rewards/accuracy_reward/mean": 0.18303571781143546, "rewards/accuracy_reward/std": 0.33333151042461395, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 1114 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 889.5580596923828, "completions/mean_terminated_length": 803.1257476806641, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.33305951758643865, "grad_norm": 0.1670326143503189, "kl": 0.7607421875, "learning_rate": 1.64778755560607e-05, "loss": 0.0373, "num_tokens": 560751054.0, "reward": 0.6467634290456772, "reward_std": 0.1395229883491993, "rewards/accuracy_reward/mean": 0.14955357275903225, "rewards/accuracy_reward/std": 0.34894050657749176, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 1115 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 877.6406707763672, "completions/mean_terminated_length": 763.9841003417969, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.3333582256739601, "grad_norm": 0.3621433973312378, "kl": 1.017578125, "learning_rate": 1.6469561525348576e-05, "loss": 0.0448, "num_tokens": 561209069.0, "reward": 0.720982164144516, "reward_std": 0.1450625155121088, "rewards/accuracy_reward/mean": 0.22767856833525002, "rewards/accuracy_reward/std": 0.3494602646678686, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1116 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 880.044677734375, "completions/mean_terminated_length": 780.6125793457031, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.3336569337614816, "grad_norm": 0.3555218577384949, "kl": 1.1591796875, "learning_rate": 1.646123979642964e-05, "loss": 0.0577, "num_tokens": 561675345.0, "reward": 0.6138393133878708, "reward_std": 0.1438250057399273, "rewards/accuracy_reward/mean": 0.1205357164144516, "rewards/accuracy_reward/std": 0.31689247861504555, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 1117 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 831.8080749511719, "completions/mean_terminated_length": 760.981689453125, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "epoch": 0.33395564184900306, "grad_norm": 0.2542336583137512, "kl": 0.857421875, "learning_rate": 1.645291037920602e-05, "loss": 0.0378, "num_tokens": 562119595.0, "reward": 0.6143973469734192, "reward_std": 0.0656950706616044, "rewards/accuracy_reward/mean": 0.11830357019789517, "rewards/accuracy_reward/std": 0.278475534170866, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1118 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 870.2500305175781, "completions/mean_terminated_length": 774.7891387939453, "completions/min_length": 405.75, "completions/min_terminated_length": 405.75, "epoch": 0.33425434993652453, "grad_norm": 0.15880289673805237, "kl": 1.125, "learning_rate": 1.6444573283588977e-05, "loss": 0.0581, "num_tokens": 562578123.0, "reward": 0.6188616305589676, "reward_std": 0.09554241970181465, "rewards/accuracy_reward/mean": 0.12276785960420966, "rewards/accuracy_reward/std": 0.2887749709188938, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.026178478728979826, "step": 1119 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 887.9464721679688, "completions/mean_terminated_length": 793.3388824462891, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "epoch": 0.334553058024046, "grad_norm": 0.21302258968353271, "kl": 1.0830078125, "learning_rate": 1.6436228519498924e-05, "loss": 0.0494, "num_tokens": 563042899.0, "reward": 0.5407366305589676, "reward_std": 0.06670059077441692, "rewards/accuracy_reward/mean": 0.044642857974395156, "rewards/accuracy_reward/std": 0.1864124909043312, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 1120 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37946428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 858.2277221679688, "completions/mean_terminated_length": 758.1955413818359, "completions/min_length": 362.75, "completions/min_terminated_length": 362.75, "epoch": 0.3348517661115675, "grad_norm": 0.22980719804763794, "kl": 0.77099609375, "learning_rate": 1.6427876096865394e-05, "loss": 0.0519, "num_tokens": 563497497.0, "reward": 0.6305803954601288, "reward_std": 0.16349168680608273, "rewards/accuracy_reward/mean": 0.13392857019789517, "rewards/accuracy_reward/std": 0.3068733178079128, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.023462072014808655, "step": 1121 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 906.5223541259766, "completions/mean_terminated_length": 801.6924896240234, "completions/min_length": 470.5, "completions/min_terminated_length": 470.5, "epoch": 0.33515047419908894, "grad_norm": 0.16931824386119843, "kl": 0.8876953125, "learning_rate": 1.641951602562703e-05, "loss": 0.0409, "num_tokens": 563978035.0, "reward": 0.6434151977300644, "reward_std": 0.07680896134115756, "rewards/accuracy_reward/mean": 0.14732143026776612, "rewards/accuracy_reward/std": 0.312490863725543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 1122 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 899.013427734375, "completions/mean_terminated_length": 791.4577789306641, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "epoch": 0.3354491822866104, "grad_norm": 0.14910461008548737, "kl": 0.748046875, "learning_rate": 1.6411148315731583e-05, "loss": 0.0378, "num_tokens": 564458777.0, "reward": 0.7131696790456772, "reward_std": 0.14220299664884806, "rewards/accuracy_reward/mean": 0.21651785308495164, "rewards/accuracy_reward/std": 0.3717498295009136, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.020272783935070038, "step": 1123 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 901.6763763427734, "completions/mean_terminated_length": 777.2015533447266, "completions/min_length": 392.5, "completions/min_terminated_length": 392.5, "epoch": 0.3357478903741319, "grad_norm": 0.16412968933582306, "kl": 0.6103515625, "learning_rate": 1.6402772977135885e-05, "loss": 0.0349, "num_tokens": 564935736.0, "reward": 0.655691996216774, "reward_std": 0.11107615381479263, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.3411780409514904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 1124 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 884.3571929931641, "completions/mean_terminated_length": 782.0290222167969, "completions/min_length": 392.25, "completions/min_terminated_length": 392.25, "epoch": 0.33604659846165336, "grad_norm": 0.13723710179328918, "kl": 0.46240234375, "learning_rate": 1.639439001980585e-05, "loss": 0.026, "num_tokens": 565410392.0, "reward": 0.7399553954601288, "reward_std": 0.09324068017303944, "rewards/accuracy_reward/mean": 0.24107142351567745, "rewards/accuracy_reward/std": 0.35779909789562225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4988839328289032, "rewards/tag_count_reward/std": 0.011811389587819576, "step": 1125 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 920.8839721679688, "completions/mean_terminated_length": 828.5800933837891, "completions/min_length": 530.75, "completions/min_terminated_length": 530.75, "epoch": 0.3363453065491748, "grad_norm": 0.20023669302463531, "kl": 0.61474609375, "learning_rate": 1.6385999453716453e-05, "loss": 0.0322, "num_tokens": 565901172.0, "reward": 0.680245578289032, "reward_std": 0.18554893881082535, "rewards/accuracy_reward/mean": 0.18526785634458065, "rewards/accuracy_reward/std": 0.3711295947432518, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.029593830928206444, "step": 1126 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 955.2076263427734, "completions/mean_terminated_length": 845.5673980712891, "completions/min_length": 499.0, "completions/min_terminated_length": 499.0, "epoch": 0.3366440146366963, "grad_norm": 0.16430805623531342, "kl": 0.8544921875, "learning_rate": 1.6377601288851733e-05, "loss": 0.0404, "num_tokens": 566405201.0, "reward": 0.5625000149011612, "reward_std": 0.09295632876455784, "rewards/accuracy_reward/mean": 0.06696428637951612, "rewards/accuracy_reward/std": 0.25027383491396904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1127 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 929.7969207763672, "completions/mean_terminated_length": 819.5548095703125, "completions/min_length": 517.5, "completions/min_terminated_length": 517.5, "epoch": 0.33694272272421777, "grad_norm": 0.19943960011005402, "kl": 0.552001953125, "learning_rate": 1.636919553520476e-05, "loss": 0.0252, "num_tokens": 566897750.0, "reward": 0.6093750447034836, "reward_std": 0.07332266867160797, "rewards/accuracy_reward/mean": 0.11383928591385484, "rewards/accuracy_reward/std": 0.29888781532645226, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.02769277011975646, "step": 1128 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 962.7545013427734, "completions/mean_terminated_length": 858.8228454589844, "completions/min_length": 607.5, "completions/min_terminated_length": 607.5, "epoch": 0.33724143081173924, "grad_norm": 0.3489118218421936, "kl": 0.6650390625, "learning_rate": 1.636078220277764e-05, "loss": 0.0343, "num_tokens": 567399816.0, "reward": 0.5714285969734192, "reward_std": 0.0781879248097539, "rewards/accuracy_reward/mean": 0.07812499930150807, "rewards/accuracy_reward/std": 0.24999831430613995, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.039929782040417194, "step": 1129 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6674107142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 958.779052734375, "completions/mean_terminated_length": 831.2880096435547, "completions/min_length": 518.25, "completions/min_terminated_length": 518.25, "epoch": 0.3375401388992607, "grad_norm": 0.33887624740600586, "kl": 1.0068359375, "learning_rate": 1.6352361301581496e-05, "loss": 0.0503, "num_tokens": 567907397.0, "reward": 0.6093750298023224, "reward_std": 0.15729648619890213, "rewards/accuracy_reward/mean": 0.11830357182770967, "rewards/accuracy_reward/std": 0.30206459388136864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04620361328125, "step": 1130 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 973.2321929931641, "completions/mean_terminated_length": 863.872314453125, "completions/min_length": 505.0, "completions/min_terminated_length": 505.0, "epoch": 0.3378388469867822, "grad_norm": 0.22096821665763855, "kl": 1.1611328125, "learning_rate": 1.6343932841636455e-05, "loss": 0.0475, "num_tokens": 568419997.0, "reward": 0.6434151977300644, "reward_std": 0.15254483744502068, "rewards/accuracy_reward/mean": 0.1540178577415645, "rewards/accuracy_reward/std": 0.3337695896625519, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04848270770162344, "step": 1131 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 931.5067443847656, "completions/mean_terminated_length": 822.169921875, "completions/min_length": 421.0, "completions/min_terminated_length": 421.0, "epoch": 0.33813755507430365, "grad_norm": 0.21652433276176453, "kl": 1.322265625, "learning_rate": 1.6335496832971643e-05, "loss": 0.0613, "num_tokens": 568914256.0, "reward": 0.5731027126312256, "reward_std": 0.1051727756857872, "rewards/accuracy_reward/mean": 0.0825892873108387, "rewards/accuracy_reward/std": 0.22301917523145676, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04517605667933822, "step": 1132 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6428571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 950.763427734375, "completions/mean_terminated_length": 820.3840942382812, "completions/min_length": 535.75, "completions/min_terminated_length": 535.75, "epoch": 0.3384362631618251, "grad_norm": 0.21778501570224762, "kl": 1.5390625, "learning_rate": 1.6327053285625164e-05, "loss": 0.0768, "num_tokens": 569412678.0, "reward": 0.6099330633878708, "reward_std": 0.12298447452485561, "rewards/accuracy_reward/mean": 0.12834821385331452, "rewards/accuracy_reward/std": 0.3027957510203123, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04820432187989354, "step": 1133 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7433035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 985.0893402099609, "completions/mean_terminated_length": 868.4406433105469, "completions/min_length": 623.25, "completions/min_terminated_length": 623.25, "epoch": 0.3387349712493466, "grad_norm": 0.21586698293685913, "kl": 2.212890625, "learning_rate": 1.631860220964409e-05, "loss": 0.1021, "num_tokens": 569933374.0, "reward": 0.5708705484867096, "reward_std": 0.1444654669612646, "rewards/accuracy_reward/mean": 0.08705357229337096, "rewards/accuracy_reward/std": 0.2555474378168583, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06139749940484762, "step": 1134 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6674107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 956.1964721679688, "completions/mean_terminated_length": 825.2686309814453, "completions/min_length": 552.25, "completions/min_terminated_length": 552.25, "epoch": 0.33903367933686807, "grad_norm": 0.27819395065307617, "kl": 2.578125, "learning_rate": 1.631014361508446e-05, "loss": 0.1223, "num_tokens": 570437446.0, "reward": 0.6116071790456772, "reward_std": 0.1460524955764413, "rewards/accuracy_reward/mean": 0.1272321455180645, "rewards/accuracy_reward/std": 0.21752525120973587, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.05948500894010067, "step": 1135 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6450892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 957.3951416015625, "completions/mean_terminated_length": 829.5968627929688, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "epoch": 0.33933238742438954, "grad_norm": 0.6876345872879028, "kl": 2.58203125, "learning_rate": 1.6301677512011248e-05, "loss": 0.114, "num_tokens": 570935847.0, "reward": 0.6657366305589676, "reward_std": 0.21819109842181206, "rewards/accuracy_reward/mean": 0.18080357648432255, "rewards/accuracy_reward/std": 0.3707078546285629, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05952764302492142, "step": 1136 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6763392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 956.0312805175781, "completions/mean_terminated_length": 811.2698059082031, "completions/min_length": 510.0, "completions/min_terminated_length": 510.0, "epoch": 0.339631095511911, "grad_norm": 0.44277969002723694, "kl": 3.15625, "learning_rate": 1.6293203910498375e-05, "loss": 0.1467, "num_tokens": 571437685.0, "reward": 0.5507812723517418, "reward_std": 0.12618192844092846, "rewards/accuracy_reward/mean": 0.06919642887078226, "rewards/accuracy_reward/std": 0.2217214722186327, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.0654362803325057, "step": 1137 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7209821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 976.6361846923828, "completions/mean_terminated_length": 855.5017395019531, "completions/min_length": 556.5, "completions/min_terminated_length": 556.5, "epoch": 0.3399298035994325, "grad_norm": 0.6207345128059387, "kl": 3.3046875, "learning_rate": 1.6284722820628677e-05, "loss": 0.1418, "num_tokens": 571943778.0, "reward": 0.616629496216774, "reward_std": 0.2113164458423853, "rewards/accuracy_reward/mean": 0.14062499580904841, "rewards/accuracy_reward/std": 0.30575818195939064, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.476004458963871, "rewards/tag_count_reward/std": 0.0736341904848814, "step": 1138 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6361607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 970.0937957763672, "completions/mean_terminated_length": 876.4572906494141, "completions/min_length": 544.0, "completions/min_terminated_length": 544.0, "epoch": 0.34022851168695395, "grad_norm": 0.2464570552110672, "kl": 2.78125, "learning_rate": 1.6276234252493903e-05, "loss": 0.125, "num_tokens": 572455964.0, "reward": 0.6662946790456772, "reward_std": 0.19272996112704277, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.37203576415777206, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946492433548, "rewards/tag_count_reward/std": 0.06946705374866724, "step": 1139 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 960.3683624267578, "completions/mean_terminated_length": 842.3243408203125, "completions/min_length": 523.25, "completions/min_terminated_length": 523.25, "epoch": 0.3405272197744754, "grad_norm": 0.1715593934059143, "kl": 1.7373046875, "learning_rate": 1.6267738216194698e-05, "loss": 0.0824, "num_tokens": 572956433.0, "reward": 0.698660746216774, "reward_std": 0.2119433656334877, "rewards/accuracy_reward/mean": 0.2120535708963871, "rewards/accuracy_reward/std": 0.4097251817584038, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05366284120827913, "step": 1140 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 957.4598693847656, "completions/mean_terminated_length": 823.8240966796875, "completions/min_length": 523.0, "completions/min_terminated_length": 523.0, "epoch": 0.3408259278619969, "grad_norm": 0.4459607005119324, "kl": 2.314453125, "learning_rate": 1.6259234721840595e-05, "loss": 0.1159, "num_tokens": 573457919.0, "reward": 0.6378348469734192, "reward_std": 0.2086355034261942, "rewards/accuracy_reward/mean": 0.15848214272409678, "rewards/accuracy_reward/std": 0.33738911151885986, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526828289032, "rewards/tag_count_reward/std": 0.06770225707441568, "step": 1141 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 930.2545013427734, "completions/mean_terminated_length": 814.2411956787109, "completions/min_length": 440.75, "completions/min_terminated_length": 440.75, "epoch": 0.3411246359495183, "grad_norm": 0.5101600289344788, "kl": 1.82421875, "learning_rate": 1.6250723779549998e-05, "loss": 0.0881, "num_tokens": 573942401.0, "reward": 0.5273437798023224, "reward_std": 0.09220761712640524, "rewards/accuracy_reward/mean": 0.04464285681024194, "rewards/accuracy_reward/std": 0.17182650417089462, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06370250228792429, "step": 1142 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 929.8795013427734, "completions/mean_terminated_length": 802.584228515625, "completions/min_length": 490.25, "completions/min_terminated_length": 490.25, "epoch": 0.3414233440370398, "grad_norm": 0.6176790595054626, "kl": 2.130859375, "learning_rate": 1.624220539945018e-05, "loss": 0.1057, "num_tokens": 574433675.0, "reward": 0.6177455633878708, "reward_std": 0.15499610267579556, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.2935466766357422, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526753783226, "rewards/tag_count_reward/std": 0.06874802615493536, "step": 1143 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 926.0893249511719, "completions/mean_terminated_length": 818.6533050537109, "completions/min_length": 513.0, "completions/min_terminated_length": 513.0, "epoch": 0.34172205212456125, "grad_norm": 0.2666276693344116, "kl": 1.99609375, "learning_rate": 1.623367959167726e-05, "loss": 0.1122, "num_tokens": 574928451.0, "reward": 0.726004496216774, "reward_std": 0.22178341075778008, "rewards/accuracy_reward/mean": 0.2433035746216774, "rewards/accuracy_reward/std": 0.422061525285244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06259806919842958, "step": 1144 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 931.1540374755859, "completions/mean_terminated_length": 818.3754425048828, "completions/min_length": 492.5, "completions/min_terminated_length": 492.5, "epoch": 0.3420207602120827, "grad_norm": 0.20261335372924805, "kl": 1.708984375, "learning_rate": 1.6225146366376198e-05, "loss": 0.0734, "num_tokens": 575421896.0, "reward": 0.6646205633878708, "reward_std": 0.13506414741277695, "rewards/accuracy_reward/mean": 0.1763392840512097, "rewards/accuracy_reward/std": 0.31963128596544266, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05163817573338747, "step": 1145 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 948.4553985595703, "completions/mean_terminated_length": 847.8722381591797, "completions/min_length": 497.25, "completions/min_terminated_length": 497.25, "epoch": 0.3423194682996042, "grad_norm": 0.3914085328578949, "kl": 2.044921875, "learning_rate": 1.6216605733700776e-05, "loss": 0.0889, "num_tokens": 575926420.0, "reward": 0.575334832072258, "reward_std": 0.17823823541402817, "rewards/accuracy_reward/mean": 0.08705357019789517, "rewards/accuracy_reward/std": 0.25885444693267345, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05133028328418732, "step": 1146 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 921.2411193847656, "completions/mean_terminated_length": 815.3421630859375, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "epoch": 0.34261817638712566, "grad_norm": 0.19310462474822998, "kl": 2.21484375, "learning_rate": 1.6208057703813595e-05, "loss": 0.1077, "num_tokens": 576406576.0, "reward": 0.5887277126312256, "reward_std": 0.1205439493060112, "rewards/accuracy_reward/mean": 0.10119047458283603, "rewards/accuracy_reward/std": 0.24995805509388447, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.0513117304071784, "step": 1147 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47321428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 897.7857513427734, "completions/mean_terminated_length": 787.4417877197266, "completions/min_length": 495.75, "completions/min_terminated_length": 495.75, "epoch": 0.34291688447464713, "grad_norm": 0.1882879137992859, "kl": 1.849609375, "learning_rate": 1.6199502286886053e-05, "loss": 0.0933, "num_tokens": 576878496.0, "reward": 0.7047991305589676, "reward_std": 0.1474168337881565, "rewards/accuracy_reward/mean": 0.21428571082651615, "rewards/accuracy_reward/std": 0.39974429458379745, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.03951727692037821, "step": 1148 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 896.372802734375, "completions/mean_terminated_length": 794.6578216552734, "completions/min_length": 415.5, "completions/min_terminated_length": 415.5, "epoch": 0.3432155925621686, "grad_norm": 0.27229127287864685, "kl": 1.857421875, "learning_rate": 1.6190939493098344e-05, "loss": 0.0869, "num_tokens": 577356263.0, "reward": 0.632254496216774, "reward_std": 0.10388631373643875, "rewards/accuracy_reward/mean": 0.1428571455180645, "rewards/accuracy_reward/std": 0.3488954082131386, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 1149 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 884.4978179931641, "completions/mean_terminated_length": 789.6206359863281, "completions/min_length": 450.5, "completions/min_terminated_length": 450.5, "epoch": 0.3435143006496901, "grad_norm": 0.18614037334918976, "kl": 2.142578125, "learning_rate": 1.618236933263943e-05, "loss": 0.11, "num_tokens": 577824326.0, "reward": 0.6300223618745804, "reward_std": 0.18529093079268932, "rewards/accuracy_reward/mean": 0.14285714039579034, "rewards/accuracy_reward/std": 0.3172171749174595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05466675851494074, "step": 1150 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4821428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 915.5603179931641, "completions/mean_terminated_length": 809.4633026123047, "completions/min_length": 447.5, "completions/min_terminated_length": 447.5, "epoch": 0.34381300873721155, "grad_norm": 0.17903773486614227, "kl": 2.03515625, "learning_rate": 1.6173791815707053e-05, "loss": 0.1056, "num_tokens": 578301489.0, "reward": 0.5803571566939354, "reward_std": 0.1278518782928586, "rewards/accuracy_reward/mean": 0.0915178582072258, "rewards/accuracy_reward/std": 0.1904653012752533, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051463617011904716, "step": 1151 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 892.3036041259766, "completions/mean_terminated_length": 793.37939453125, "completions/min_length": 379.75, "completions/min_terminated_length": 379.75, "epoch": 0.344111716824733, "grad_norm": 0.40329501032829285, "kl": 1.498046875, "learning_rate": 1.6165206952507694e-05, "loss": 0.0769, "num_tokens": 578775113.0, "reward": 0.6289062649011612, "reward_std": 0.14658620208501816, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.33632371574640274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818479284644, "step": 1152 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 910.0223693847656, "completions/mean_terminated_length": 800.9905242919922, "completions/min_length": 425.25, "completions/min_terminated_length": 425.25, "epoch": 0.3444104249122545, "grad_norm": 0.2205485999584198, "kl": 1.34228515625, "learning_rate": 1.6156614753256583e-05, "loss": 0.0666, "num_tokens": 579263555.0, "reward": 0.5837053805589676, "reward_std": 0.1019542757421732, "rewards/accuracy_reward/mean": 0.10007440205663443, "rewards/accuracy_reward/std": 0.246732659637928, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04168279003351927, "step": 1153 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43080357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 896.1674499511719, "completions/mean_terminated_length": 799.7625427246094, "completions/min_length": 463.25, "completions/min_terminated_length": 463.25, "epoch": 0.34470913299977596, "grad_norm": 0.26642778515815735, "kl": 1.01806640625, "learning_rate": 1.6148015228177682e-05, "loss": 0.0574, "num_tokens": 579739470.0, "reward": 0.7706473618745804, "reward_std": 0.1675033662468195, "rewards/accuracy_reward/mean": 0.276785708963871, "rewards/accuracy_reward/std": 0.4357788413763046, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03258697595447302, "step": 1154 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38616071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 878.6964569091797, "completions/mean_terminated_length": 787.7542724609375, "completions/min_length": 493.25, "completions/min_terminated_length": 493.25, "epoch": 0.34500784108729743, "grad_norm": 0.17784084379673004, "kl": 1.2470703125, "learning_rate": 1.6139408387503667e-05, "loss": 0.0653, "num_tokens": 580202630.0, "reward": 0.6623884290456772, "reward_std": 0.16393635049462318, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.3744606673717499, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 1155 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 912.0134429931641, "completions/mean_terminated_length": 821.0511779785156, "completions/min_length": 529.0, "completions/min_terminated_length": 529.0, "epoch": 0.3453065491748189, "grad_norm": 0.23130445182323456, "kl": 0.765625, "learning_rate": 1.6130794241475912e-05, "loss": 0.0418, "num_tokens": 580682060.0, "reward": 0.5786830633878708, "reward_std": 0.11635785666294396, "rewards/accuracy_reward/mean": 0.08482142863795161, "rewards/accuracy_reward/std": 0.2216135412454605, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1156 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 890.5692291259766, "completions/mean_terminated_length": 797.1463165283203, "completions/min_length": 436.0, "completions/min_terminated_length": 436.0, "epoch": 0.34560525726234037, "grad_norm": 0.49319368600845337, "kl": 0.9560546875, "learning_rate": 1.6122172800344494e-05, "loss": 0.0636, "num_tokens": 581152875.0, "reward": 0.6127232313156128, "reward_std": 0.15512312203645706, "rewards/accuracy_reward/mean": 0.12053571455180645, "rewards/accuracy_reward/std": 0.2746327817440033, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 1157 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 892.0446929931641, "completions/mean_terminated_length": 820.8533172607422, "completions/min_length": 433.25, "completions/min_terminated_length": 433.25, "epoch": 0.34590396534986184, "grad_norm": 0.16516415774822235, "kl": 0.908203125, "learning_rate": 1.6113544074368166e-05, "loss": 0.0544, "num_tokens": 581624527.0, "reward": 0.7516741305589676, "reward_std": 0.18194666504859924, "rewards/accuracy_reward/mean": 0.2589285708963871, "rewards/accuracy_reward/std": 0.4392261952161789, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040314854588359594, "step": 1158 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 875.4286041259766, "completions/mean_terminated_length": 780.1727447509766, "completions/min_length": 421.75, "completions/min_terminated_length": 421.75, "epoch": 0.3462026734373833, "grad_norm": 0.5535374879837036, "kl": 1.1787109375, "learning_rate": 1.6104908073814348e-05, "loss": 0.0772, "num_tokens": 582083183.0, "reward": 0.6395089775323868, "reward_std": 0.15375462360680103, "rewards/accuracy_reward/mean": 0.14955357508733869, "rewards/accuracy_reward/std": 0.2937432676553726, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 1159 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 898.1942291259766, "completions/mean_terminated_length": 803.322265625, "completions/min_length": 489.75, "completions/min_terminated_length": 489.75, "epoch": 0.3465013815249048, "grad_norm": 0.3448908030986786, "kl": 1.513671875, "learning_rate": 1.6096264808959123e-05, "loss": 0.0728, "num_tokens": 582559110.0, "reward": 0.5541294887661934, "reward_std": 0.08137914631515741, "rewards/accuracy_reward/mean": 0.06696428637951612, "rewards/accuracy_reward/std": 0.1595880165696144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 1160 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 906.294677734375, "completions/mean_terminated_length": 784.5973663330078, "completions/min_length": 463.0, "completions/min_terminated_length": 463.0, "epoch": 0.34680008961242625, "grad_norm": 0.32876256108283997, "kl": 1.58203125, "learning_rate": 1.608761429008721e-05, "loss": 0.0707, "num_tokens": 583035370.0, "reward": 0.6473214477300644, "reward_std": 0.14510491490364075, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.35614214837551117, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 1161 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 882.8370971679688, "completions/mean_terminated_length": 779.7332763671875, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.3470987976999477, "grad_norm": 0.20299819111824036, "kl": 1.9140625, "learning_rate": 1.607895652749196e-05, "loss": 0.0995, "num_tokens": 583498833.0, "reward": 0.6428571790456772, "reward_std": 0.1418066006153822, "rewards/accuracy_reward/mean": 0.15178571501746774, "rewards/accuracy_reward/std": 0.3292398937046528, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04420433798804879, "step": 1162 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875000000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 904.4486999511719, "completions/mean_terminated_length": 797.1531829833984, "completions/min_length": 460.5, "completions/min_terminated_length": 460.5, "epoch": 0.3473975057874692, "grad_norm": 0.6648603081703186, "kl": 2.873046875, "learning_rate": 1.6070291531475342e-05, "loss": 0.1286, "num_tokens": 583982170.0, "reward": 0.6562500298023224, "reward_std": 0.18911293894052505, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.3645346686244011, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.05924179404973984, "step": 1163 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 895.6674499511719, "completions/mean_terminated_length": 793.8209075927734, "completions/min_length": 351.5, "completions/min_terminated_length": 351.5, "epoch": 0.34769621387499067, "grad_norm": 0.4501063823699951, "kl": 2.58984375, "learning_rate": 1.606161931234795e-05, "loss": 0.1212, "num_tokens": 584454517.0, "reward": 0.5926339626312256, "reward_std": 0.13237760588526726, "rewards/accuracy_reward/mean": 0.10491071548312902, "rewards/accuracy_reward/std": 0.29361283779144287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05140746245160699, "step": 1164 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 895.7545013427734, "completions/mean_terminated_length": 792.8564910888672, "completions/min_length": 438.75, "completions/min_terminated_length": 438.75, "epoch": 0.34799492196251214, "grad_norm": 0.2873550057411194, "kl": 2.06640625, "learning_rate": 1.6052939880428942e-05, "loss": 0.0933, "num_tokens": 584930615.0, "reward": 0.6534598469734192, "reward_std": 0.13276489078998566, "rewards/accuracy_reward/mean": 0.16517856903374195, "rewards/accuracy_reward/std": 0.35294684022665024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05048930738121271, "step": 1165 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 885.8370971679688, "completions/mean_terminated_length": 794.5144958496094, "completions/min_length": 449.5, "completions/min_terminated_length": 449.5, "epoch": 0.3482936300500336, "grad_norm": 0.21013061702251434, "kl": 1.6640625, "learning_rate": 1.604425324604609e-05, "loss": 0.0825, "num_tokens": 585396590.0, "reward": 0.7968750447034836, "reward_std": 0.18781765177845955, "rewards/accuracy_reward/mean": 0.3058035746216774, "rewards/accuracy_reward/std": 0.45129717886447906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.042937278281897306, "step": 1166 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 931.8013763427734, "completions/mean_terminated_length": 828.1944274902344, "completions/min_length": 516.25, "completions/min_terminated_length": 516.25, "epoch": 0.3485923381375551, "grad_norm": 0.17912088334560394, "kl": 1.1591796875, "learning_rate": 1.6035559419535714e-05, "loss": 0.0609, "num_tokens": 585878645.0, "reward": 0.5998884290456772, "reward_std": 0.1397099643945694, "rewards/accuracy_reward/mean": 0.10714285634458065, "rewards/accuracy_reward/std": 0.30954253673553467, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 1167 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 897.9799346923828, "completions/mean_terminated_length": 817.4649810791016, "completions/min_length": 455.5, "completions/min_terminated_length": 455.5, "epoch": 0.34889104622507655, "grad_norm": 0.18221262097358704, "kl": 1.1376953125, "learning_rate": 1.6026858411242704e-05, "loss": 0.0552, "num_tokens": 586349516.0, "reward": 0.6378348469734192, "reward_std": 0.1585574634373188, "rewards/accuracy_reward/mean": 0.14508928637951612, "rewards/accuracy_reward/std": 0.33844296634197235, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04205985926091671, "step": 1168 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 843.6585235595703, "completions/mean_terminated_length": 755.7166900634766, "completions/min_length": 384.25, "completions/min_terminated_length": 384.25, "epoch": 0.349189754312598, "grad_norm": 0.2907485067844391, "kl": 1.16796875, "learning_rate": 1.6018150231520486e-05, "loss": 0.062, "num_tokens": 586795523.0, "reward": 0.6160714477300644, "reward_std": 0.10261120181530714, "rewards/accuracy_reward/mean": 0.12500000186264515, "rewards/accuracy_reward/std": 0.33038803935050964, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.044314838480204344, "step": 1169 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 875.2835083007812, "completions/mean_terminated_length": 782.4931182861328, "completions/min_length": 468.25, "completions/min_terminated_length": 468.25, "epoch": 0.3494884624001195, "grad_norm": 0.3863403797149658, "kl": 1.390625, "learning_rate": 1.6009434890731027e-05, "loss": 0.0984, "num_tokens": 587253330.0, "reward": 0.7455357313156128, "reward_std": 0.22203389182686806, "rewards/accuracy_reward/mean": 0.2626488134264946, "rewards/accuracy_reward/std": 0.41672077029943466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051574116572737694, "step": 1170 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 874.5067443847656, "completions/mean_terminated_length": 773.7581939697266, "completions/min_length": 397.75, "completions/min_terminated_length": 397.75, "epoch": 0.34978717048764096, "grad_norm": 0.18268226087093353, "kl": 1.4716796875, "learning_rate": 1.6000712399244813e-05, "loss": 0.0792, "num_tokens": 587718869.0, "reward": 0.6941964626312256, "reward_std": 0.1670590043067932, "rewards/accuracy_reward/mean": 0.20312499813735485, "rewards/accuracy_reward/std": 0.38714759051799774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 1171 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 848.8861999511719, "completions/mean_terminated_length": 754.9463195800781, "completions/min_length": 363.5, "completions/min_terminated_length": 363.5, "epoch": 0.35008587857516243, "grad_norm": 0.20402227342128754, "kl": 1.349609375, "learning_rate": 1.5991982767440835e-05, "loss": 0.073, "num_tokens": 588167362.0, "reward": 0.654575914144516, "reward_std": 0.16240262798964977, "rewards/accuracy_reward/mean": 0.1607142873108387, "rewards/accuracy_reward/std": 0.34941166266798973, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.031825252808630466, "step": 1172 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 899.5335235595703, "completions/mean_terminated_length": 776.7665405273438, "completions/min_length": 403.25, "completions/min_terminated_length": 403.25, "epoch": 0.3503845866626839, "grad_norm": 0.19214126467704773, "kl": 1.939453125, "learning_rate": 1.5983246005706592e-05, "loss": 0.1046, "num_tokens": 588644241.0, "reward": 0.6612723469734192, "reward_std": 0.18458189442753792, "rewards/accuracy_reward/mean": 0.1718750037252903, "rewards/accuracy_reward/std": 0.36264148727059364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 1173 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 866.6786041259766, "completions/mean_terminated_length": 744.9212493896484, "completions/min_length": 390.5, "completions/min_terminated_length": 390.5, "epoch": 0.3506832947502054, "grad_norm": 0.20430268347263336, "kl": 1.6640625, "learning_rate": 1.597450212443805e-05, "loss": 0.1074, "num_tokens": 589098401.0, "reward": 0.6685268133878708, "reward_std": 0.13014075718820095, "rewards/accuracy_reward/mean": 0.17633928474970162, "rewards/accuracy_reward/std": 0.3397493492811918, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 1174 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 865.6428985595703, "completions/mean_terminated_length": 766.3651733398438, "completions/min_length": 391.75, "completions/min_terminated_length": 391.75, "epoch": 0.35098200283772685, "grad_norm": 0.259854257106781, "kl": 1.55859375, "learning_rate": 1.5965751134039665e-05, "loss": 0.0802, "num_tokens": 589555217.0, "reward": 0.7120535969734192, "reward_std": 0.14005344174802303, "rewards/accuracy_reward/mean": 0.21874999860301614, "rewards/accuracy_reward/std": 0.3591485060751438, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1175 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 899.7589569091797, "completions/mean_terminated_length": 769.58544921875, "completions/min_length": 445.75, "completions/min_terminated_length": 445.75, "epoch": 0.3512807109252483, "grad_norm": 0.32421761751174927, "kl": 1.765625, "learning_rate": 1.5956993044924334e-05, "loss": 0.1037, "num_tokens": 590027701.0, "reward": 0.6121652126312256, "reward_std": 0.10870141629129648, "rewards/accuracy_reward/mean": 0.11830357275903225, "rewards/accuracy_reward/std": 0.27082016319036484, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03752126870676875, "step": 1176 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 856.0312805175781, "completions/mean_terminated_length": 753.6677093505859, "completions/min_length": 396.5, "completions/min_terminated_length": 396.5, "epoch": 0.3515794190127698, "grad_norm": 0.22311295568943024, "kl": 2.13671875, "learning_rate": 1.5948227867513416e-05, "loss": 0.11, "num_tokens": 590489251.0, "reward": 0.6813616305589676, "reward_std": 0.14607828110456467, "rewards/accuracy_reward/mean": 0.19419642630964518, "rewards/accuracy_reward/std": 0.360374853014946, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054504433646798134, "step": 1177 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42187500000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 857.5402069091797, "completions/mean_terminated_length": 742.4010162353516, "completions/min_length": 366.5, "completions/min_terminated_length": 366.5, "epoch": 0.35187812710029126, "grad_norm": 0.4711384177207947, "kl": 2.2734375, "learning_rate": 1.593945561223669e-05, "loss": 0.1176, "num_tokens": 590948821.0, "reward": 0.680245578289032, "reward_std": 0.1913565956056118, "rewards/accuracy_reward/mean": 0.1919642835855484, "rewards/accuracy_reward/std": 0.37406614422798157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 1178 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 916.1250305175781, "completions/mean_terminated_length": 810.4590148925781, "completions/min_length": 435.25, "completions/min_terminated_length": 435.25, "epoch": 0.35217683518781273, "grad_norm": 0.33349815011024475, "kl": 1.64453125, "learning_rate": 1.5930676289532373e-05, "loss": 0.0812, "num_tokens": 591432317.0, "reward": 0.6021205633878708, "reward_std": 0.11875620856881142, "rewards/accuracy_reward/mean": 0.11160714481957257, "rewards/accuracy_reward/std": 0.27876229397952557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1179 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47991071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 890.7120819091797, "completions/mean_terminated_length": 767.4639434814453, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "epoch": 0.3524755432753342, "grad_norm": 0.579937219619751, "kl": 2.07421875, "learning_rate": 1.592188990984708e-05, "loss": 0.1176, "num_tokens": 591901180.0, "reward": 0.6093750149011612, "reward_std": 0.19341611489653587, "rewards/accuracy_reward/mean": 0.12500000232830644, "rewards/accuracy_reward/std": 0.3056153394281864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.05820113513618708, "step": 1180 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40624999999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 886.5201263427734, "completions/mean_terminated_length": 796.7925415039062, "completions/min_length": 391.25, "completions/min_terminated_length": 391.25, "epoch": 0.3527742513628557, "grad_norm": 0.32740092277526855, "kl": 1.794921875, "learning_rate": 1.5913096483635827e-05, "loss": 0.1017, "num_tokens": 592369141.0, "reward": 0.6579241454601288, "reward_std": 0.17365818284451962, "rewards/accuracy_reward/mean": 0.16741070849820971, "rewards/accuracy_reward/std": 0.34445153921842575, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04565368499606848, "step": 1181 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 889.1094207763672, "completions/mean_terminated_length": 774.3059387207031, "completions/min_length": 422.25, "completions/min_terminated_length": 422.25, "epoch": 0.35307295945037714, "grad_norm": 0.2245093584060669, "kl": 2.357421875, "learning_rate": 1.5904296021362014e-05, "loss": 0.1214, "num_tokens": 592834038.0, "reward": 0.6763392984867096, "reward_std": 0.1448053978383541, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.3813312128186226, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05820057261735201, "step": 1182 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 855.2232666015625, "completions/mean_terminated_length": 753.0362548828125, "completions/min_length": 421.25, "completions/min_terminated_length": 421.25, "epoch": 0.3533716675378986, "grad_norm": 0.1863722950220108, "kl": 1.5126953125, "learning_rate": 1.5895488533497415e-05, "loss": 0.0808, "num_tokens": 593292778.0, "reward": 0.651785746216774, "reward_std": 0.14422067254781723, "rewards/accuracy_reward/mean": 0.16294643096625805, "rewards/accuracy_reward/std": 0.3658328950405121, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.0487851407378912, "step": 1183 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 883.0402069091797, "completions/mean_terminated_length": 761.0253601074219, "completions/min_length": 415.25, "completions/min_terminated_length": 415.25, "epoch": 0.3536703756254201, "grad_norm": 0.23200228810310364, "kl": 1.744140625, "learning_rate": 1.588667403052216e-05, "loss": 0.0901, "num_tokens": 593763948.0, "reward": 0.6233258992433548, "reward_std": 0.16981963068246841, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.33824337273836136, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.044580988585948944, "step": 1184 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 925.7835083007812, "completions/mean_terminated_length": 818.2775573730469, "completions/min_length": 455.0, "completions/min_terminated_length": 455.0, "epoch": 0.3539690837129415, "grad_norm": 0.2583156228065491, "kl": 1.810546875, "learning_rate": 1.5877852522924733e-05, "loss": 0.0854, "num_tokens": 594250139.0, "reward": 0.5731027126312256, "reward_std": 0.12873821519315243, "rewards/accuracy_reward/mean": 0.08482142770662904, "rewards/accuracy_reward/std": 0.25460201874375343, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.052778348326683044, "step": 1185 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 917.9710388183594, "completions/mean_terminated_length": 789.3871917724609, "completions/min_length": 503.5, "completions/min_terminated_length": 503.5, "epoch": 0.35426779180046297, "grad_norm": 0.33029237389564514, "kl": 2.115234375, "learning_rate": 1.586902402120195e-05, "loss": 0.101, "num_tokens": 594727902.0, "reward": 0.5948661118745804, "reward_std": 0.1307857297360897, "rewards/accuracy_reward/mean": 0.10937499930150807, "rewards/accuracy_reward/std": 0.27737261168658733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05552538204938173, "step": 1186 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 919.6473846435547, "completions/mean_terminated_length": 812.5482482910156, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "epoch": 0.35456649988798444, "grad_norm": 0.3190293312072754, "kl": 1.90234375, "learning_rate": 1.586018853585894e-05, "loss": 0.0868, "num_tokens": 595207424.0, "reward": 0.6272321492433548, "reward_std": 0.1877823956310749, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3426527678966522, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04808079404756427, "step": 1187 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 880.0781707763672, "completions/mean_terminated_length": 772.2832946777344, "completions/min_length": 471.75, "completions/min_terminated_length": 471.75, "epoch": 0.3548652079755059, "grad_norm": 0.3961870074272156, "kl": 2.259765625, "learning_rate": 1.585134607740916e-05, "loss": 0.1071, "num_tokens": 595670915.0, "reward": 0.6891741454601288, "reward_std": 0.14971128106117249, "rewards/accuracy_reward/mean": 0.20089285261929035, "rewards/accuracy_reward/std": 0.3938790261745453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04914125660434365, "step": 1188 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33928571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 857.0201263427734, "completions/mean_terminated_length": 774.2197418212891, "completions/min_length": 432.75, "completions/min_terminated_length": 432.75, "epoch": 0.3551639160630274, "grad_norm": 0.5325331091880798, "kl": 2.109375, "learning_rate": 1.5842496656374347e-05, "loss": 0.1092, "num_tokens": 596124364.0, "reward": 0.6568080484867096, "reward_std": 0.16948936134576797, "rewards/accuracy_reward/mean": 0.16741071082651615, "rewards/accuracy_reward/std": 0.3700050488114357, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04868226684629917, "step": 1189 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 837.1875457763672, "completions/mean_terminated_length": 709.2516937255859, "completions/min_length": 368.75, "completions/min_terminated_length": 368.75, "epoch": 0.35546262415054886, "grad_norm": 0.23281903564929962, "kl": 1.56640625, "learning_rate": 1.5833640283284534e-05, "loss": 0.0719, "num_tokens": 596574624.0, "reward": 0.6367187798023224, "reward_std": 0.13744687475264072, "rewards/accuracy_reward/mean": 0.14508928824216127, "rewards/accuracy_reward/std": 0.3283962607383728, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 1190 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 878.1138763427734, "completions/mean_terminated_length": 765.8788909912109, "completions/min_length": 411.5, "completions/min_terminated_length": 411.5, "epoch": 0.3557613322380703, "grad_norm": 0.1919003427028656, "kl": 1.8828125, "learning_rate": 1.5824776968678024e-05, "loss": 0.0903, "num_tokens": 597043331.0, "reward": 0.6785714477300644, "reward_std": 0.17087064683437347, "rewards/accuracy_reward/mean": 0.1919642835855484, "rewards/accuracy_reward/std": 0.39047618955373764, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05434367246925831, "step": 1191 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5803571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 940.4688110351562, "completions/mean_terminated_length": 824.3534851074219, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "epoch": 0.3560600403255918, "grad_norm": 0.22206243872642517, "kl": 1.028564453125, "learning_rate": 1.5815906723101377e-05, "loss": 0.046, "num_tokens": 597543893.0, "reward": 0.6143973618745804, "reward_std": 0.13731988333165646, "rewards/accuracy_reward/mean": 0.1257440485060215, "rewards/accuracy_reward/std": 0.3261338844895363, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.037955629639327526, "step": 1192 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 891.8951263427734, "completions/mean_terminated_length": 787.9860229492188, "completions/min_length": 448.5, "completions/min_terminated_length": 448.5, "epoch": 0.35635874841311327, "grad_norm": 0.7846044898033142, "kl": 1.37109375, "learning_rate": 1.5807029557109398e-05, "loss": 0.0836, "num_tokens": 598025846.0, "reward": 0.6266741454601288, "reward_std": 0.15473352745175362, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.34839701652526855, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05713389813899994, "step": 1193 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41517857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 870.6027069091797, "completions/mean_terminated_length": 764.3482818603516, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.35665745650063474, "grad_norm": 0.2020365297794342, "kl": 1.361328125, "learning_rate": 1.579814548126514e-05, "loss": 0.0629, "num_tokens": 598493428.0, "reward": 0.7008928805589676, "reward_std": 0.1647991370409727, "rewards/accuracy_reward/mean": 0.2183779776096344, "rewards/accuracy_reward/std": 0.4054490774869919, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.049318346194922924, "step": 1194 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 891.232177734375, "completions/mean_terminated_length": 801.7333221435547, "completions/min_length": 449.0, "completions/min_terminated_length": 449.0, "epoch": 0.3569561645881562, "grad_norm": 0.504855215549469, "kl": 1.875, "learning_rate": 1.578925450613986e-05, "loss": 0.0976, "num_tokens": 598966460.0, "reward": 0.5837053805589676, "reward_std": 0.15517712011933327, "rewards/accuracy_reward/mean": 0.10267857369035482, "rewards/accuracy_reward/std": 0.2913813665509224, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06581770908087492, "step": 1195 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 938.2678985595703, "completions/mean_terminated_length": 820.1773529052734, "completions/min_length": 468.75, "completions/min_terminated_length": 468.75, "epoch": 0.3572548726756777, "grad_norm": 0.20806053280830383, "kl": 1.6904296875, "learning_rate": 1.5780356642313034e-05, "loss": 0.0786, "num_tokens": 599462916.0, "reward": 0.6238839626312256, "reward_std": 0.17781337350606918, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.33895042538642883, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05349547974765301, "step": 1196 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 887.779052734375, "completions/mean_terminated_length": 770.9423065185547, "completions/min_length": 440.5, "completions/min_terminated_length": 440.5, "epoch": 0.35755358076319915, "grad_norm": 0.254476934671402, "kl": 2.48828125, "learning_rate": 1.577145190037234e-05, "loss": 0.1235, "num_tokens": 599937025.0, "reward": 0.7075893133878708, "reward_std": 0.2204630970954895, "rewards/accuracy_reward/mean": 0.2254464328289032, "rewards/accuracy_reward/std": 0.41009606420993805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06410299427807331, "step": 1197 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 871.6205749511719, "completions/mean_terminated_length": 773.3051452636719, "completions/min_length": 399.25, "completions/min_terminated_length": 399.25, "epoch": 0.3578522888507206, "grad_norm": 0.2470523715019226, "kl": 2.140625, "learning_rate": 1.5762540290913628e-05, "loss": 0.1113, "num_tokens": 600403047.0, "reward": 0.706473246216774, "reward_std": 0.19527901336550713, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.4008224681019783, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 1198 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 882.3638763427734, "completions/mean_terminated_length": 734.5621948242188, "completions/min_length": 348.25, "completions/min_terminated_length": 348.25, "epoch": 0.3581509969382421, "grad_norm": 0.30604034662246704, "kl": 1.974609375, "learning_rate": 1.5753621824540924e-05, "loss": 0.096, "num_tokens": 600875450.0, "reward": 0.792410746216774, "reward_std": 0.1571136675775051, "rewards/accuracy_reward/mean": 0.30133928172290325, "rewards/accuracy_reward/std": 0.43610159307718277, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04605984315276146, "step": 1199 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49107142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 917.0022735595703, "completions/mean_terminated_length": 819.4858856201172, "completions/min_length": 472.0, "completions/min_terminated_length": 472.0, "epoch": 0.35844970502576357, "grad_norm": 0.22748880088329315, "kl": 2.8203125, "learning_rate": 1.5744696511866426e-05, "loss": 0.1268, "num_tokens": 601362859.0, "reward": 0.5239955633878708, "reward_std": 0.10320722218602896, "rewards/accuracy_reward/mean": 0.040178571827709675, "rewards/accuracy_reward/std": 0.16817742586135864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06139749940484762, "step": 1200 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 932.4643249511719, "completions/mean_terminated_length": 819.6523590087891, "completions/min_length": 465.0, "completions/min_terminated_length": 465.0, "epoch": 0.35874841311328504, "grad_norm": 0.7137405276298523, "kl": 3.40234375, "learning_rate": 1.573576436351046e-05, "loss": 0.154, "num_tokens": 601849451.0, "reward": 0.7031250298023224, "reward_std": 0.23614771850407124, "rewards/accuracy_reward/mean": 0.22098213713616133, "rewards/accuracy_reward/std": 0.38578758388757706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06271844450384378, "step": 1201 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 894.3594207763672, "completions/mean_terminated_length": 758.0928039550781, "completions/min_length": 442.75, "completions/min_terminated_length": 442.75, "epoch": 0.3590471212008065, "grad_norm": 0.3439663052558899, "kl": 2.2041015625, "learning_rate": 1.57268253901015e-05, "loss": 0.1053, "num_tokens": 602317852.0, "reward": 0.5731026977300644, "reward_std": 0.1037674811668694, "rewards/accuracy_reward/mean": 0.08482143003493547, "rewards/accuracy_reward/std": 0.2227826490998268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050832636654376984, "step": 1202 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 896.2745971679688, "completions/mean_terminated_length": 785.4339294433594, "completions/min_length": 453.25, "completions/min_terminated_length": 453.25, "epoch": 0.359345829288328, "grad_norm": 0.2709144949913025, "kl": 2.32421875, "learning_rate": 1.5717879602276123e-05, "loss": 0.1177, "num_tokens": 602795575.0, "reward": 0.584263414144516, "reward_std": 0.1665100511163473, "rewards/accuracy_reward/mean": 0.09821428498253226, "rewards/accuracy_reward/std": 0.27171074599027634, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05749546363949776, "step": 1203 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 932.9754943847656, "completions/mean_terminated_length": 814.4592132568359, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.35964453737584945, "grad_norm": 0.41452351212501526, "kl": 2.087890625, "learning_rate": 1.5708927010679038e-05, "loss": 0.0951, "num_tokens": 603298524.0, "reward": 0.5926339477300644, "reward_std": 0.1419474110007286, "rewards/accuracy_reward/mean": 0.1093749983701855, "rewards/accuracy_reward/std": 0.26904776506125927, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06215344648808241, "step": 1204 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 929.3370971679688, "completions/mean_terminated_length": 803.0370025634766, "completions/min_length": 454.75, "completions/min_terminated_length": 454.75, "epoch": 0.3599432454633709, "grad_norm": 0.38775262236595154, "kl": 1.74609375, "learning_rate": 1.5699967625963032e-05, "loss": 0.0911, "num_tokens": 603783843.0, "reward": 0.646205373108387, "reward_std": 0.1907917968928814, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.29981905221939087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.485491082072258, "rewards/tag_count_reward/std": 0.05699931550770998, "step": 1205 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 908.8795013427734, "completions/mean_terminated_length": 774.3454284667969, "completions/min_length": 457.5, "completions/min_terminated_length": 457.5, "epoch": 0.3602419535508924, "grad_norm": 0.2953389286994934, "kl": 2.482421875, "learning_rate": 1.5691001458788984e-05, "loss": 0.1181, "num_tokens": 604258589.0, "reward": 0.6590401977300644, "reward_std": 0.14555939473211765, "rewards/accuracy_reward/mean": 0.17633928172290325, "rewards/accuracy_reward/std": 0.3744852691888809, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06215373892337084, "step": 1206 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 934.9531860351562, "completions/mean_terminated_length": 820.2890472412109, "completions/min_length": 509.75, "completions/min_terminated_length": 509.75, "epoch": 0.36054066163841386, "grad_norm": 0.2703936994075775, "kl": 1.705078125, "learning_rate": 1.568202851982584e-05, "loss": 0.0879, "num_tokens": 604750424.0, "reward": 0.5691964477300644, "reward_std": 0.1936862487345934, "rewards/accuracy_reward/mean": 0.08035714458674192, "rewards/accuracy_reward/std": 0.26593760401010513, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 1207 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44642857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 882.5781707763672, "completions/mean_terminated_length": 768.6019897460938, "completions/min_length": 357.5, "completions/min_terminated_length": 357.5, "epoch": 0.36083936972593533, "grad_norm": 0.19603806734085083, "kl": 1.888671875, "learning_rate": 1.5673048819750604e-05, "loss": 0.0975, "num_tokens": 605212363.0, "reward": 0.6768973469734192, "reward_std": 0.19161364436149597, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3924490064382553, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 1208 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 909.2053985595703, "completions/mean_terminated_length": 797.5891723632812, "completions/min_length": 430.75, "completions/min_terminated_length": 430.75, "epoch": 0.3611380778134568, "grad_norm": 0.2932203412055969, "kl": 1.46142578125, "learning_rate": 1.566406236924833e-05, "loss": 0.0711, "num_tokens": 605693015.0, "reward": 0.6579241454601288, "reward_std": 0.15480593964457512, "rewards/accuracy_reward/mean": 0.16741071408614516, "rewards/accuracy_reward/std": 0.33074766397476196, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04036805871874094, "step": 1209 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 885.4531555175781, "completions/mean_terminated_length": 748.5458679199219, "completions/min_length": 338.75, "completions/min_terminated_length": 338.75, "epoch": 0.3614367859009783, "grad_norm": 0.2591165602207184, "kl": 1.39013671875, "learning_rate": 1.5655069179012096e-05, "loss": 0.0765, "num_tokens": 606157170.0, "reward": 0.7047991454601288, "reward_std": 0.20700563490390778, "rewards/accuracy_reward/mean": 0.20982142817229033, "rewards/accuracy_reward/std": 0.3606291189789772, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.029593830928206444, "step": 1210 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 877.9353179931641, "completions/mean_terminated_length": 739.7650604248047, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "epoch": 0.36173549398849975, "grad_norm": 0.22241252660751343, "kl": 1.3310546875, "learning_rate": 1.5646069259743007e-05, "loss": 0.0673, "num_tokens": 606617973.0, "reward": 0.6143973618745804, "reward_std": 0.15381797403097153, "rewards/accuracy_reward/mean": 0.1205357164144516, "rewards/accuracy_reward/std": 0.3233230784535408, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03752126870676875, "step": 1211 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 877.4219207763672, "completions/mean_terminated_length": 744.0467834472656, "completions/min_length": 440.75, "completions/min_terminated_length": 440.75, "epoch": 0.3620342020760212, "grad_norm": 0.2297709584236145, "kl": 1.5390625, "learning_rate": 1.5637062622150168e-05, "loss": 0.0717, "num_tokens": 607089346.0, "reward": 0.6238839626312256, "reward_std": 0.16994713805615902, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.32652072235941887, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 1212 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 853.8147888183594, "completions/mean_terminated_length": 729.3622894287109, "completions/min_length": 363.25, "completions/min_terminated_length": 363.25, "epoch": 0.3623329101635427, "grad_norm": 0.25192588567733765, "kl": 2.021484375, "learning_rate": 1.5628049276950687e-05, "loss": 0.1076, "num_tokens": 607549103.0, "reward": 0.681919664144516, "reward_std": 0.18182164430618286, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.393671877682209, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.049703214317560196, "step": 1213 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 869.9554138183594, "completions/mean_terminated_length": 748.4151458740234, "completions/min_length": 382.5, "completions/min_terminated_length": 382.5, "epoch": 0.36263161825106416, "grad_norm": 0.25016269087791443, "kl": 1.6953125, "learning_rate": 1.5619029234869646e-05, "loss": 0.0976, "num_tokens": 608007099.0, "reward": 0.647879496216774, "reward_std": 0.19358355179429054, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.34330489858984947, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04288960574194789, "step": 1214 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 812.6853179931641, "completions/mean_terminated_length": 701.3864135742188, "completions/min_length": 335.5, "completions/min_terminated_length": 335.5, "epoch": 0.36293032633858563, "grad_norm": 0.6910520195960999, "kl": 1.826171875, "learning_rate": 1.56100025066401e-05, "loss": 0.0916, "num_tokens": 608434302.0, "reward": 0.7120535969734192, "reward_std": 0.2026572860777378, "rewards/accuracy_reward/mean": 0.21875000186264515, "rewards/accuracy_reward/std": 0.4055922403931618, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 1215 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 903.5670013427734, "completions/mean_terminated_length": 774.7703552246094, "completions/min_length": 349.75, "completions/min_terminated_length": 349.75, "epoch": 0.3632290344261071, "grad_norm": 0.35997700691223145, "kl": 1.802734375, "learning_rate": 1.5600969103003056e-05, "loss": 0.0969, "num_tokens": 608917756.0, "reward": 0.6261160969734192, "reward_std": 0.12891943380236626, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.27117954194545746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043574148789048195, "step": 1216 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 909.5893249511719, "completions/mean_terminated_length": 773.9602203369141, "completions/min_length": 411.25, "completions/min_terminated_length": 411.25, "epoch": 0.36352774251362857, "grad_norm": 0.2770021855831146, "kl": 1.5380859375, "learning_rate": 1.5591929034707468e-05, "loss": 0.0741, "num_tokens": 609395268.0, "reward": 0.6757812798023224, "reward_std": 0.18738648854196072, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.3853269889950752, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03541599866002798, "step": 1217 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 846.6027069091797, "completions/mean_terminated_length": 758.4801025390625, "completions/min_length": 374.75, "completions/min_terminated_length": 374.75, "epoch": 0.36382645060115004, "grad_norm": 0.4797694683074951, "kl": 1.3115234375, "learning_rate": 1.558288231251022e-05, "loss": 0.0729, "num_tokens": 609841154.0, "reward": 0.6540178805589676, "reward_std": 0.13486536592245102, "rewards/accuracy_reward/mean": 0.160714291036129, "rewards/accuracy_reward/std": 0.3559816926717758, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03934345254674554, "step": 1218 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 875.0870819091797, "completions/mean_terminated_length": 767.8504638671875, "completions/min_length": 453.25, "completions/min_terminated_length": 453.25, "epoch": 0.3641251586886715, "grad_norm": 0.31780606508255005, "kl": 1.265625, "learning_rate": 1.5573828947176114e-05, "loss": 0.0543, "num_tokens": 610305481.0, "reward": 0.6568080633878708, "reward_std": 0.14815166033804417, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.36302707344293594, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03258697595447302, "step": 1219 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 827.7031555175781, "completions/mean_terminated_length": 751.4629516601562, "completions/min_length": 337.25, "completions/min_terminated_length": 337.25, "epoch": 0.364423866776193, "grad_norm": 0.2038334310054779, "kl": 1.0, "learning_rate": 1.5564768949477848e-05, "loss": 0.0595, "num_tokens": 610746852.0, "reward": 0.7455357611179352, "reward_std": 0.1898653469979763, "rewards/accuracy_reward/mean": 0.2503720249515027, "rewards/accuracy_reward/std": 0.3792366571724415, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.032084173522889614, "step": 1220 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 916.2924499511719, "completions/mean_terminated_length": 842.5365447998047, "completions/min_length": 558.75, "completions/min_terminated_length": 558.75, "epoch": 0.36472257486371445, "grad_norm": 0.15209606289863586, "kl": 0.9072265625, "learning_rate": 1.5555702330196024e-05, "loss": 0.0436, "num_tokens": 611240183.0, "reward": 0.6590402126312256, "reward_std": 0.17735589481890202, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.3579200655221939, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 1221 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 883.8460235595703, "completions/mean_terminated_length": 800.6555786132812, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "epoch": 0.3650212829512359, "grad_norm": 0.30180802941322327, "kl": 1.3330078125, "learning_rate": 1.554662910011912e-05, "loss": 0.0732, "num_tokens": 611707154.0, "reward": 0.6651785969734192, "reward_std": 0.18280698917806149, "rewards/accuracy_reward/mean": 0.17410713993012905, "rewards/accuracy_reward/std": 0.3623902350664139, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194884091616, "step": 1222 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 823.450927734375, "completions/mean_terminated_length": 760.1213836669922, "completions/min_length": 320.25, "completions/min_terminated_length": 320.25, "epoch": 0.3653199910387574, "grad_norm": 0.40303346514701843, "kl": 1.0029296875, "learning_rate": 1.5537549270043474e-05, "loss": 0.0632, "num_tokens": 612150716.0, "reward": 0.7031250298023224, "reward_std": 0.1152288168668747, "rewards/accuracy_reward/mean": 0.21242559258826077, "rewards/accuracy_reward/std": 0.3310838919132948, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03934345254674554, "step": 1223 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 851.1875305175781, "completions/mean_terminated_length": 777.2120208740234, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "epoch": 0.36561869912627887, "grad_norm": 0.26775333285331726, "kl": 1.2783203125, "learning_rate": 1.5528462850773284e-05, "loss": 0.0552, "num_tokens": 612610592.0, "reward": 0.6841518133878708, "reward_std": 0.19838659837841988, "rewards/accuracy_reward/mean": 0.19196428544819355, "rewards/accuracy_reward/std": 0.3785649314522743, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.03613344579935074, "step": 1224 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2589285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 830.3080749511719, "completions/mean_terminated_length": 768.8986663818359, "completions/min_length": 394.25, "completions/min_terminated_length": 394.25, "epoch": 0.36591740721380034, "grad_norm": 0.2587185800075531, "kl": 1.3876953125, "learning_rate": 1.5519369853120584e-05, "loss": 0.0742, "num_tokens": 613056906.0, "reward": 0.6662946790456772, "reward_std": 0.15419126860797405, "rewards/accuracy_reward/mean": 0.1741071380674839, "rewards/accuracy_reward/std": 0.3656275123357773, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04008414130657911, "step": 1225 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.22767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 838.8437805175781, "completions/mean_terminated_length": 786.4695739746094, "completions/min_length": 423.5, "completions/min_terminated_length": 423.5, "epoch": 0.3662161153013218, "grad_norm": 0.6150221824645996, "kl": 1.7109375, "learning_rate": 1.5510270287905243e-05, "loss": 0.0884, "num_tokens": 613507732.0, "reward": 0.5703125223517418, "reward_std": 0.10908063501119614, "rewards/accuracy_reward/mean": 0.09188987873494625, "rewards/accuracy_reward/std": 0.22338243573904037, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 1226 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2544642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 841.7098693847656, "completions/mean_terminated_length": 779.7271575927734, "completions/min_length": 424.5, "completions/min_terminated_length": 424.5, "epoch": 0.3665148233888433, "grad_norm": 0.23986592888832092, "kl": 2.154296875, "learning_rate": 1.5501164165954935e-05, "loss": 0.1214, "num_tokens": 613951730.0, "reward": 0.690848246216774, "reward_std": 0.18775740265846252, "rewards/accuracy_reward/mean": 0.21205356903374195, "rewards/accuracy_reward/std": 0.38336507976055145, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 1227 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2566964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 833.0223541259766, "completions/mean_terminated_length": 767.9814147949219, "completions/min_length": 449.75, "completions/min_terminated_length": 449.75, "epoch": 0.3668135314763647, "grad_norm": 0.4641565680503845, "kl": 1.953125, "learning_rate": 1.5492051498105144e-05, "loss": 0.0934, "num_tokens": 614394908.0, "reward": 0.6110491305589676, "reward_std": 0.12610813602805138, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.3167544938623905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 1228 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 856.2232360839844, "completions/mean_terminated_length": 782.5327911376953, "completions/min_length": 488.0, "completions/min_terminated_length": 488.0, "epoch": 0.36711223956388617, "grad_norm": 0.5578910708427429, "kl": 2.28515625, "learning_rate": 1.548293229519914e-05, "loss": 0.1157, "num_tokens": 614855824.0, "reward": 0.737723246216774, "reward_std": 0.2258499525487423, "rewards/accuracy_reward/mean": 0.2477678544819355, "rewards/accuracy_reward/std": 0.4225282147526741, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 1229 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2678571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 859.9174499511719, "completions/mean_terminated_length": 801.5349884033203, "completions/min_length": 468.75, "completions/min_terminated_length": 468.75, "epoch": 0.36741094765140764, "grad_norm": 0.5628220438957214, "kl": 2.37890625, "learning_rate": 1.547380656808797e-05, "loss": 0.1136, "num_tokens": 615306923.0, "reward": 0.6434152126312256, "reward_std": 0.1433033999055624, "rewards/accuracy_reward/mean": 0.15401786006987095, "rewards/accuracy_reward/std": 0.32451484724879265, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.0494418740272522, "step": 1230 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 866.0156707763672, "completions/mean_terminated_length": 787.7854614257812, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "epoch": 0.3677096557389291, "grad_norm": 0.20259752869606018, "kl": 1.8515625, "learning_rate": 1.5464674327630437e-05, "loss": 0.0954, "num_tokens": 615761042.0, "reward": 0.6149553805589676, "reward_std": 0.15034009143710136, "rewards/accuracy_reward/mean": 0.1250000020954758, "rewards/accuracy_reward/std": 0.2769377138465643, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764320462942, "step": 1231 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 890.9420166015625, "completions/mean_terminated_length": 789.7291412353516, "completions/min_length": 389.75, "completions/min_terminated_length": 389.75, "epoch": 0.3680083638264506, "grad_norm": 0.21559913456439972, "kl": 1.625, "learning_rate": 1.5455535584693105e-05, "loss": 0.0779, "num_tokens": 616223720.0, "reward": 0.6305803805589676, "reward_std": 0.16501597687602043, "rewards/accuracy_reward/mean": 0.14285714039579034, "rewards/accuracy_reward/std": 0.3068162016570568, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05248269159346819, "step": 1232 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 877.9643402099609, "completions/mean_terminated_length": 795.1191711425781, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "epoch": 0.36830707191397205, "grad_norm": 0.7870690226554871, "kl": 1.5546875, "learning_rate": 1.5446390350150272e-05, "loss": 0.0898, "num_tokens": 616694056.0, "reward": 0.737723246216774, "reward_std": 0.16713106259703636, "rewards/accuracy_reward/mean": 0.2499999962747097, "rewards/accuracy_reward/std": 0.43022316694259644, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 1233 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 892.5826263427734, "completions/mean_terminated_length": 793.5787048339844, "completions/min_length": 388.25, "completions/min_terminated_length": 388.25, "epoch": 0.3686057800014935, "grad_norm": 0.2926877439022064, "kl": 1.71875, "learning_rate": 1.543723863488396e-05, "loss": 0.0948, "num_tokens": 617170957.0, "reward": 0.6741071790456772, "reward_std": 0.19181258231401443, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.38033800572156906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.055475836619734764, "step": 1234 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 921.8393402099609, "completions/mean_terminated_length": 819.4379730224609, "completions/min_length": 469.75, "completions/min_terminated_length": 469.75, "epoch": 0.368904488089015, "grad_norm": 0.455965518951416, "kl": 1.4453125, "learning_rate": 1.5428080449783898e-05, "loss": 0.0851, "num_tokens": 617661365.0, "reward": 0.617745578289032, "reward_std": 0.1641537994146347, "rewards/accuracy_reward/mean": 0.12946428847499192, "rewards/accuracy_reward/std": 0.2927661668509245, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 1235 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 928.529052734375, "completions/mean_terminated_length": 821.4326477050781, "completions/min_length": 455.5, "completions/min_terminated_length": 455.5, "epoch": 0.36920319617653646, "grad_norm": 0.6791447401046753, "kl": 1.6923828125, "learning_rate": 1.5418915805747518e-05, "loss": 0.0892, "num_tokens": 618141938.0, "reward": 0.607142873108387, "reward_std": 0.12100072577595711, "rewards/accuracy_reward/mean": 0.12276785378344357, "rewards/accuracy_reward/std": 0.2861786689609289, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.05946850869804621, "step": 1236 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 924.1116485595703, "completions/mean_terminated_length": 814.9029388427734, "completions/min_length": 389.5, "completions/min_terminated_length": 389.5, "epoch": 0.36950190426405793, "grad_norm": 0.5076403617858887, "kl": 1.892578125, "learning_rate": 1.5409744713679942e-05, "loss": 0.0958, "num_tokens": 618624052.0, "reward": 0.6372768133878708, "reward_std": 0.1863220576196909, "rewards/accuracy_reward/mean": 0.15401785913854837, "rewards/accuracy_reward/std": 0.30381766706705093, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06243238039314747, "step": 1237 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 920.8348693847656, "completions/mean_terminated_length": 793.3602294921875, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 0.3698006123515794, "grad_norm": 0.27519041299819946, "kl": 2.619140625, "learning_rate": 1.5400567184493953e-05, "loss": 0.1294, "num_tokens": 619108346.0, "reward": 0.6445312798023224, "reward_std": 0.15305832028388977, "rewards/accuracy_reward/mean": 0.1607142873108387, "rewards/accuracy_reward/std": 0.36382535845041275, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.059261033311486244, "step": 1238 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 876.8281707763672, "completions/mean_terminated_length": 757.2893218994141, "completions/min_length": 380.25, "completions/min_terminated_length": 380.25, "epoch": 0.3700993204391009, "grad_norm": 0.4605879783630371, "kl": 3.126953125, "learning_rate": 1.5391383229110005e-05, "loss": 0.1571, "num_tokens": 619574157.0, "reward": 0.6646205633878708, "reward_std": 0.1555009838193655, "rewards/accuracy_reward/mean": 0.18303570849820971, "rewards/accuracy_reward/std": 0.35869547724723816, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848171710968, "rewards/tag_count_reward/std": 0.06361658219248056, "step": 1239 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 910.2835235595703, "completions/mean_terminated_length": 783.4105987548828, "completions/min_length": 344.75, "completions/min_terminated_length": 344.75, "epoch": 0.37039802852662235, "grad_norm": 0.9188393950462341, "kl": 4.5625, "learning_rate": 1.538219285845619e-05, "loss": 0.2128, "num_tokens": 620053836.0, "reward": 0.5998884215950966, "reward_std": 0.16775053553283215, "rewards/accuracy_reward/mean": 0.1246279755141586, "rewards/accuracy_reward/std": 0.25865335017442703, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4771205335855484, "rewards/tag_count_reward/std": 0.07068470306694508, "step": 1240 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 915.7924499511719, "completions/mean_terminated_length": 779.9913330078125, "completions/min_length": 314.25, "completions/min_terminated_length": 314.25, "epoch": 0.3706967366141438, "grad_norm": 0.5468993782997131, "kl": 3.89453125, "learning_rate": 1.5372996083468242e-05, "loss": 0.1845, "num_tokens": 620536319.0, "reward": 0.6205357313156128, "reward_std": 0.18130242079496384, "rewards/accuracy_reward/mean": 0.14285714644938707, "rewards/accuracy_reward/std": 0.3283647820353508, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4776785671710968, "rewards/tag_count_reward/std": 0.0697621526196599, "step": 1241 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 939.7210235595703, "completions/mean_terminated_length": 804.273193359375, "completions/min_length": 333.5, "completions/min_terminated_length": 333.5, "epoch": 0.3709954447016653, "grad_norm": 0.4545626938343048, "kl": 3.201171875, "learning_rate": 1.5363792915089505e-05, "loss": 0.1447, "num_tokens": 621033858.0, "reward": 0.633928582072258, "reward_std": 0.1769878026098013, "rewards/accuracy_reward/mean": 0.15401786006987095, "rewards/accuracy_reward/std": 0.3597414121031761, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4799107164144516, "rewards/tag_count_reward/std": 0.06501297932118177, "step": 1242 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 900.1607513427734, "completions/mean_terminated_length": 792.49609375, "completions/min_length": 236.75, "completions/min_terminated_length": 236.75, "epoch": 0.37129415278918676, "grad_norm": 0.2511630952358246, "kl": 2.87109375, "learning_rate": 1.535458336427094e-05, "loss": 0.1343, "num_tokens": 621506314.0, "reward": 0.6367187947034836, "reward_std": 0.18287185579538345, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.3544863499701023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366007566452, "rewards/tag_count_reward/std": 0.07179340533912182, "step": 1243 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 853.8236999511719, "completions/mean_terminated_length": 703.7367401123047, "completions/min_length": 210.0, "completions/min_terminated_length": 210.0, "epoch": 0.37159286087670823, "grad_norm": 1.0760222673416138, "kl": 4.25390625, "learning_rate": 1.53453674419711e-05, "loss": 0.202, "num_tokens": 621965819.0, "reward": 0.550223246216774, "reward_std": 0.15160807967185974, "rewards/accuracy_reward/mean": 0.08035714225843549, "rewards/accuracy_reward/std": 0.2557261809706688, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4698660671710968, "rewards/tag_count_reward/std": 0.08033232297748327, "step": 1244 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5803571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 910.2455749511719, "completions/mean_terminated_length": 765.3728485107422, "completions/min_length": 222.25, "completions/min_terminated_length": 222.25, "epoch": 0.3718915689642297, "grad_norm": 0.6191796660423279, "kl": 2.28125, "learning_rate": 1.533614515915612e-05, "loss": 0.1188, "num_tokens": 622440633.0, "reward": 0.5758928880095482, "reward_std": 0.1401064768433571, "rewards/accuracy_reward/mean": 0.1004464291036129, "rewards/accuracy_reward/std": 0.19884613901376724, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4754464253783226, "rewards/tag_count_reward/std": 0.07497264258563519, "step": 1245 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.5, "completions/mean_length": 920.3170013427734, "completions/mean_terminated_length": 756.1631317138672, "completions/min_length": 286.5, "completions/min_terminated_length": 286.5, "epoch": 0.3721902770517512, "grad_norm": 0.8946438431739807, "kl": 1.921875, "learning_rate": 1.532691652679969e-05, "loss": 0.1057, "num_tokens": 622930103.0, "reward": 0.6612723469734192, "reward_std": 0.174334904178977, "rewards/accuracy_reward/mean": 0.18080356670543551, "rewards/accuracy_reward/std": 0.33047283813357353, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48046875, "rewards/tag_count_reward/std": 0.06697986274957657, "step": 1246 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 927.4464721679688, "completions/mean_terminated_length": 814.7980499267578, "completions/min_length": 250.25, "completions/min_terminated_length": 250.25, "epoch": 0.37248898513927264, "grad_norm": 0.734836757183075, "kl": 1.5859375, "learning_rate": 1.5317681555883087e-05, "loss": 0.0914, "num_tokens": 623415391.0, "reward": 0.6562500298023224, "reward_std": 0.18442139867693186, "rewards/accuracy_reward/mean": 0.16964285634458065, "rewards/accuracy_reward/std": 0.2850732207298279, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05488624330610037, "step": 1247 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 867.4129791259766, "completions/mean_terminated_length": 764.1756134033203, "completions/min_length": 320.75, "completions/min_terminated_length": 320.75, "epoch": 0.3727876932267941, "grad_norm": 0.6255983710289001, "kl": 1.7314453125, "learning_rate": 1.5308440257395095e-05, "loss": 0.098, "num_tokens": 623883384.0, "reward": 0.6780134290456772, "reward_std": 0.17380706034600735, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.36072514206171036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05420170119032264, "step": 1248 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 872.1272583007812, "completions/mean_terminated_length": 753.3241729736328, "completions/min_length": 216.5, "completions/min_terminated_length": 216.5, "epoch": 0.3730864013143156, "grad_norm": 0.41539227962493896, "kl": 1.693359375, "learning_rate": 1.529919264233205e-05, "loss": 0.0945, "num_tokens": 624344465.0, "reward": 0.652901828289032, "reward_std": 0.11595615092664957, "rewards/accuracy_reward/mean": 0.1629464253783226, "rewards/accuracy_reward/std": 0.3076270893216133, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04863459337502718, "step": 1249 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 854.2232513427734, "completions/mean_terminated_length": 745.8946838378906, "completions/min_length": 303.75, "completions/min_terminated_length": 303.75, "epoch": 0.37338510940183706, "grad_norm": 0.5557080507278442, "kl": 2.41796875, "learning_rate": 1.5289938721697795e-05, "loss": 0.1239, "num_tokens": 624799941.0, "reward": 0.6724330484867096, "reward_std": 0.15641267783939838, "rewards/accuracy_reward/mean": 0.18303571129217744, "rewards/accuracy_reward/std": 0.34742752090096474, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.044561849907040596, "step": 1250 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 914.5335235595703, "completions/mean_terminated_length": 782.7138519287109, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "epoch": 0.3736838174893585, "grad_norm": 0.4973854124546051, "kl": 2.88671875, "learning_rate": 1.528067850650368e-05, "loss": 0.1403, "num_tokens": 625281508.0, "reward": 0.5948661118745804, "reward_std": 0.15767976082861423, "rewards/accuracy_reward/mean": 0.10937500046566129, "rewards/accuracy_reward/std": 0.28081436082720757, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.058194358833134174, "step": 1251 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 844.2232360839844, "completions/mean_terminated_length": 747.8035125732422, "completions/min_length": 269.5, "completions/min_terminated_length": 269.5, "epoch": 0.37398252557688, "grad_norm": 0.3680846691131592, "kl": 1.912109375, "learning_rate": 1.5271412007768543e-05, "loss": 0.1004, "num_tokens": 625731720.0, "reward": 0.6757812947034836, "reward_std": 0.1871692407876253, "rewards/accuracy_reward/mean": 0.1830357164144516, "rewards/accuracy_reward/std": 0.38686653226614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196588039398, "step": 1252 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 862.4152221679688, "completions/mean_terminated_length": 771.02880859375, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "epoch": 0.37428123366440147, "grad_norm": 0.4947282373905182, "kl": 2.052734375, "learning_rate": 1.5262139236518695e-05, "loss": 0.1014, "num_tokens": 626184930.0, "reward": 0.5524553805589676, "reward_std": 0.08981577539816499, "rewards/accuracy_reward/mean": 0.05803571455180645, "rewards/accuracy_reward/std": 0.1718190498650074, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.036314870696514845, "step": 1253 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 849.5893249511719, "completions/mean_terminated_length": 728.4317626953125, "completions/min_length": 267.5, "completions/min_terminated_length": 267.5, "epoch": 0.37457994175192294, "grad_norm": 0.5003321766853333, "kl": 1.4794921875, "learning_rate": 1.5252860203787923e-05, "loss": 0.0853, "num_tokens": 626635610.0, "reward": 0.6724330633878708, "reward_std": 0.16307808458805084, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.37341081351041794, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1254 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 906.7812805175781, "completions/mean_terminated_length": 815.0483703613281, "completions/min_length": 519.25, "completions/min_terminated_length": 519.25, "epoch": 0.3748786498394444, "grad_norm": 0.363534539937973, "kl": 1.5751953125, "learning_rate": 1.5243574920617445e-05, "loss": 0.0763, "num_tokens": 627111352.0, "reward": 0.658482164144516, "reward_std": 0.09893092326819897, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.30961688607931137, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02827909868210554, "step": 1255 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 878.7879943847656, "completions/mean_terminated_length": 737.4110565185547, "completions/min_length": 279.25, "completions/min_terminated_length": 279.25, "epoch": 0.3751773579269659, "grad_norm": 0.2078246772289276, "kl": 1.548828125, "learning_rate": 1.523428339805594e-05, "loss": 0.0847, "num_tokens": 627583849.0, "reward": 0.6065848618745804, "reward_std": 0.12060939520597458, "rewards/accuracy_reward/mean": 0.11383928311988711, "rewards/accuracy_reward/std": 0.3005780577659607, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03546962048858404, "step": 1256 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 885.4933471679688, "completions/mean_terminated_length": 780.1998443603516, "completions/min_length": 363.5, "completions/min_terminated_length": 363.5, "epoch": 0.37547606601448735, "grad_norm": 0.13713939487934113, "kl": 0.8681640625, "learning_rate": 1.5224985647159489e-05, "loss": 0.0407, "num_tokens": 628052118.0, "reward": 0.576450914144516, "reward_std": 0.09842831455171108, "rewards/accuracy_reward/mean": 0.08035714155994356, "rewards/accuracy_reward/std": 0.23418232053518295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 1257 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 881.2991485595703, "completions/mean_terminated_length": 768.8102569580078, "completions/min_length": 285.0, "completions/min_terminated_length": 285.0, "epoch": 0.3757747741020088, "grad_norm": 0.26791054010391235, "kl": 0.974609375, "learning_rate": 1.5215681678991603e-05, "loss": 0.0477, "num_tokens": 628519356.0, "reward": 0.6233259290456772, "reward_std": 0.10484071681275964, "rewards/accuracy_reward/mean": 0.145461305975914, "rewards/accuracy_reward/std": 0.29100501537323, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.038415491580963135, "step": 1258 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 907.2098693847656, "completions/mean_terminated_length": 802.7059936523438, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.3760734821895303, "grad_norm": 0.1814344972372055, "kl": 0.7919921875, "learning_rate": 1.5206371504623175e-05, "loss": 0.0485, "num_tokens": 628997082.0, "reward": 0.7081473469734192, "reward_std": 0.14124685525894165, "rewards/accuracy_reward/mean": 0.21205357578583062, "rewards/accuracy_reward/std": 0.33113059774041176, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.02676480822265148, "step": 1259 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 887.1920013427734, "completions/mean_terminated_length": 741.33935546875, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "epoch": 0.37637219027705177, "grad_norm": 0.14753644168376923, "kl": 1.15478515625, "learning_rate": 1.5197055135132495e-05, "loss": 0.059, "num_tokens": 629464304.0, "reward": 0.6590402126312256, "reward_std": 0.1418459948617965, "rewards/accuracy_reward/mean": 0.1651785708963871, "rewards/accuracy_reward/std": 0.311752550303936, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03532243426889181, "step": 1260 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 844.8527221679688, "completions/mean_terminated_length": 744.1477966308594, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "epoch": 0.37667089836457324, "grad_norm": 0.3803357183933258, "kl": 0.9580078125, "learning_rate": 1.5187732581605217e-05, "loss": 0.0679, "num_tokens": 629920798.0, "reward": 0.6344866454601288, "reward_std": 0.1534517128020525, "rewards/accuracy_reward/mean": 0.14062500139698386, "rewards/accuracy_reward/std": 0.316734679043293, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03782916208729148, "step": 1261 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 885.3281555175781, "completions/mean_terminated_length": 789.1977233886719, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "epoch": 0.3769696064520947, "grad_norm": 0.2886386513710022, "kl": 0.73486328125, "learning_rate": 1.5178403855134357e-05, "loss": 0.0434, "num_tokens": 630385697.0, "reward": 0.5758928805589676, "reward_std": 0.09801226039417088, "rewards/accuracy_reward/mean": 0.0803571417927742, "rewards/accuracy_reward/std": 0.23248441517353058, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027692769188433886, "step": 1262 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 914.4174499511719, "completions/mean_terminated_length": 774.3317413330078, "completions/min_length": 372.75, "completions/min_terminated_length": 372.75, "epoch": 0.3772683145396162, "grad_norm": 0.2391556203365326, "kl": 0.9521484375, "learning_rate": 1.5169068966820275e-05, "loss": 0.0548, "num_tokens": 630864524.0, "reward": 0.6891741305589676, "reward_std": 0.23008642345666885, "rewards/accuracy_reward/mean": 0.20982142724096775, "rewards/accuracy_reward/std": 0.380741149187088, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 1263 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 877.9710235595703, "completions/mean_terminated_length": 766.7500152587891, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "epoch": 0.37756702262713765, "grad_norm": 0.19604533910751343, "kl": 1.19482421875, "learning_rate": 1.515972792777067e-05, "loss": 0.0783, "num_tokens": 631323199.0, "reward": 0.6143973469734192, "reward_std": 0.16170762479305267, "rewards/accuracy_reward/mean": 0.1205357126891613, "rewards/accuracy_reward/std": 0.27577266842126846, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.031825252808630466, "step": 1264 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 929.9821929931641, "completions/mean_terminated_length": 818.9141845703125, "completions/min_length": 467.0, "completions/min_terminated_length": 467.0, "epoch": 0.3778657307146591, "grad_norm": 0.16319404542446136, "kl": 0.837890625, "learning_rate": 1.5150380749100545e-05, "loss": 0.044, "num_tokens": 631820631.0, "reward": 0.5786830633878708, "reward_std": 0.10121542913839221, "rewards/accuracy_reward/mean": 0.0825892835855484, "rewards/accuracy_reward/std": 0.23499742150306702, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 1265 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 915.1786193847656, "completions/mean_terminated_length": 803.8117218017578, "completions/min_length": 487.5, "completions/min_terminated_length": 487.5, "epoch": 0.3781644388021806, "grad_norm": 0.2748429477214813, "kl": 1.1728515625, "learning_rate": 1.5141027441932217e-05, "loss": 0.0519, "num_tokens": 632303575.0, "reward": 0.7483259290456772, "reward_std": 0.15035432763397694, "rewards/accuracy_reward/mean": 0.25446428544819355, "rewards/accuracy_reward/std": 0.4144709035754204, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 1266 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 906.4710235595703, "completions/mean_terminated_length": 809.1566009521484, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "epoch": 0.37846314688970206, "grad_norm": 0.21853891015052795, "kl": 1.857421875, "learning_rate": 1.5131668017395304e-05, "loss": 0.0974, "num_tokens": 632781802.0, "reward": 0.6657366305589676, "reward_std": 0.12005366943776608, "rewards/accuracy_reward/mean": 0.1763392894063145, "rewards/accuracy_reward/std": 0.3056818451732397, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 1267 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 900.6652069091797, "completions/mean_terminated_length": 792.8187866210938, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "epoch": 0.37876185497722353, "grad_norm": 0.3178013265132904, "kl": 1.560546875, "learning_rate": 1.5122302486626687e-05, "loss": 0.0913, "num_tokens": 633257172.0, "reward": 0.7762277126312256, "reward_std": 0.23787051253020763, "rewards/accuracy_reward/mean": 0.2834821445867419, "rewards/accuracy_reward/std": 0.39683081209659576, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 1268 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 908.7991485595703, "completions/mean_terminated_length": 782.63037109375, "completions/min_length": 414.25, "completions/min_terminated_length": 414.25, "epoch": 0.379060563064745, "grad_norm": 0.2901570498943329, "kl": 1.12890625, "learning_rate": 1.511293086077052e-05, "loss": 0.0444, "num_tokens": 633738586.0, "reward": 0.679129496216774, "reward_std": 0.13580838218331337, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.36170896142721176, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937574505806, "rewards/tag_count_reward/std": 0.029367767740041018, "step": 1269 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 929.1585388183594, "completions/mean_terminated_length": 801.009765625, "completions/min_length": 444.25, "completions/min_terminated_length": 444.25, "epoch": 0.3793592711522665, "grad_norm": 0.3794296085834503, "kl": 1.73388671875, "learning_rate": 1.5103553150978219e-05, "loss": 0.0695, "num_tokens": 634225953.0, "reward": 0.707589328289032, "reward_std": 0.17341961339116096, "rewards/accuracy_reward/mean": 0.21651786006987095, "rewards/accuracy_reward/std": 0.4012888967990875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04029684793204069, "step": 1270 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 898.0223541259766, "completions/mean_terminated_length": 788.6300506591797, "completions/min_length": 403.75, "completions/min_terminated_length": 403.75, "epoch": 0.3796579792397879, "grad_norm": 0.34232717752456665, "kl": 1.80859375, "learning_rate": 1.509416936840842e-05, "loss": 0.0916, "num_tokens": 634700731.0, "reward": 0.6467634290456772, "reward_std": 0.13242628797888756, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.35350678861141205, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 1271 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 921.9129791259766, "completions/mean_terminated_length": 812.6255950927734, "completions/min_length": 361.75, "completions/min_terminated_length": 361.75, "epoch": 0.37995668732730936, "grad_norm": 0.348152220249176, "kl": 1.6240234375, "learning_rate": 1.5084779524227e-05, "loss": 0.0784, "num_tokens": 635187812.0, "reward": 0.750558078289032, "reward_std": 0.23755230754613876, "rewards/accuracy_reward/mean": 0.2589285671710968, "rewards/accuracy_reward/std": 0.4298956021666527, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1272 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 911.6428833007812, "completions/mean_terminated_length": 765.7025451660156, "completions/min_length": 378.75, "completions/min_terminated_length": 378.75, "epoch": 0.38025539541483083, "grad_norm": 0.23599711060523987, "kl": 1.80078125, "learning_rate": 1.5075383629607043e-05, "loss": 0.0873, "num_tokens": 635660452.0, "reward": 0.68917416036129, "reward_std": 0.14425427839159966, "rewards/accuracy_reward/mean": 0.2008928544819355, "rewards/accuracy_reward/std": 0.3932851627469063, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 1273 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 918.6004943847656, "completions/mean_terminated_length": 808.96044921875, "completions/min_length": 400.75, "completions/min_terminated_length": 400.75, "epoch": 0.3805541035023523, "grad_norm": 0.27776971459388733, "kl": 1.3212890625, "learning_rate": 1.5065981695728837e-05, "loss": 0.0659, "num_tokens": 636143313.0, "reward": 0.6796875447034836, "reward_std": 0.23151268810033798, "rewards/accuracy_reward/mean": 0.18973214365541935, "rewards/accuracy_reward/std": 0.3821932002902031, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 1274 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 935.5781707763672, "completions/mean_terminated_length": 824.7442016601562, "completions/min_length": 432.75, "completions/min_terminated_length": 432.75, "epoch": 0.3808528115898738, "grad_norm": 0.256893515586853, "kl": 1.560546875, "learning_rate": 1.5056573733779848e-05, "loss": 0.0757, "num_tokens": 636635492.0, "reward": 0.614397332072258, "reward_std": 0.1332504553720355, "rewards/accuracy_reward/mean": 0.12723214481957257, "rewards/accuracy_reward/std": 0.2831819038838148, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 1275 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 924.3549652099609, "completions/mean_terminated_length": 783.5350494384766, "completions/min_length": 432.5, "completions/min_terminated_length": 432.5, "epoch": 0.38115151967739525, "grad_norm": 0.3951913118362427, "kl": 1.509765625, "learning_rate": 1.5047159754954721e-05, "loss": 0.0794, "num_tokens": 637121619.0, "reward": 0.5675223395228386, "reward_std": 0.1328315045684576, "rewards/accuracy_reward/mean": 0.08035714365541935, "rewards/accuracy_reward/std": 0.22856499627232552, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.055264041759073734, "step": 1276 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 924.3437957763672, "completions/mean_terminated_length": 824.8324279785156, "completions/min_length": 487.75, "completions/min_terminated_length": 487.75, "epoch": 0.3814502277649167, "grad_norm": 0.45746609568595886, "kl": 1.580078125, "learning_rate": 1.5037739770455263e-05, "loss": 0.076, "num_tokens": 637612349.0, "reward": 0.5686384215950966, "reward_std": 0.1294950796291232, "rewards/accuracy_reward/mean": 0.08258928498253226, "rewards/accuracy_reward/std": 0.21338041871786118, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.05577457416802645, "step": 1277 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 886.435302734375, "completions/mean_terminated_length": 802.3150177001953, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "epoch": 0.3817489358524382, "grad_norm": 0.521135687828064, "kl": 1.021484375, "learning_rate": 1.5028313791490424e-05, "loss": 0.0655, "num_tokens": 638081968.0, "reward": 0.7059152126312256, "reward_std": 0.15929356031119823, "rewards/accuracy_reward/mean": 0.2142857126891613, "rewards/accuracy_reward/std": 0.40765298157930374, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.044580988585948944, "step": 1278 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 906.5647888183594, "completions/mean_terminated_length": 814.9726409912109, "completions/min_length": 493.5, "completions/min_terminated_length": 493.5, "epoch": 0.38204764393995966, "grad_norm": 0.3525238037109375, "kl": 1.78125, "learning_rate": 1.501888182927628e-05, "loss": 0.0909, "num_tokens": 638555245.0, "reward": 0.6082589626312256, "reward_std": 0.17712851613759995, "rewards/accuracy_reward/mean": 0.12053571455180645, "rewards/accuracy_reward/std": 0.323215052485466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.054059810005128384, "step": 1279 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 879.2299652099609, "completions/mean_terminated_length": 772.3558959960938, "completions/min_length": 428.5, "completions/min_terminated_length": 428.5, "epoch": 0.38234635202748113, "grad_norm": 0.3117935061454773, "kl": 1.87353515625, "learning_rate": 1.5009443895036036e-05, "loss": 0.1049, "num_tokens": 639021396.0, "reward": 0.7204241305589676, "reward_std": 0.1958073191344738, "rewards/accuracy_reward/mean": 0.23214285541325808, "rewards/accuracy_reward/std": 0.383254736661911, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.044885930605232716, "step": 1280 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 918.4553985595703, "completions/mean_terminated_length": 828.3139038085938, "completions/min_length": 454.25, "completions/min_terminated_length": 454.25, "epoch": 0.3826450601150026, "grad_norm": 0.2747930884361267, "kl": 2.05078125, "learning_rate": 1.5000000000000002e-05, "loss": 0.098, "num_tokens": 639510192.0, "reward": 0.7081473618745804, "reward_std": 0.19382456690073013, "rewards/accuracy_reward/mean": 0.21875000186264515, "rewards/accuracy_reward/std": 0.38428349420428276, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04875553119927645, "step": 1281 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 875.9955749511719, "completions/mean_terminated_length": 785.9923400878906, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.38294376820252407, "grad_norm": 0.46006807684898376, "kl": 1.984375, "learning_rate": 1.4990550155405579e-05, "loss": 0.1012, "num_tokens": 639969518.0, "reward": 0.6852678954601288, "reward_std": 0.15062957257032394, "rewards/accuracy_reward/mean": 0.19940476259216666, "rewards/accuracy_reward/std": 0.34656479582190514, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 1282 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 893.5714721679688, "completions/mean_terminated_length": 790.8937835693359, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "epoch": 0.38324247629004554, "grad_norm": 0.1906944364309311, "kl": 2.08203125, "learning_rate": 1.4981094372497243e-05, "loss": 0.1063, "num_tokens": 640441790.0, "reward": 0.6914062947034836, "reward_std": 0.16667982190847397, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.39021211117506027, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 1283 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37946428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 887.5089721679688, "completions/mean_terminated_length": 806.1019439697266, "completions/min_length": 428.5, "completions/min_terminated_length": 428.5, "epoch": 0.383541184377567, "grad_norm": 0.24072757363319397, "kl": 1.728515625, "learning_rate": 1.4971632662526545e-05, "loss": 0.0852, "num_tokens": 640909410.0, "reward": 0.537946455180645, "reward_std": 0.08948200289160013, "rewards/accuracy_reward/mean": 0.046875, "rewards/accuracy_reward/std": 0.1459379866719246, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04620361328125, "step": 1284 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29464285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 851.6317291259766, "completions/mean_terminated_length": 780.5050964355469, "completions/min_length": 378.25, "completions/min_terminated_length": 378.25, "epoch": 0.3838398924650885, "grad_norm": 0.44136765599250793, "kl": 2.3779296875, "learning_rate": 1.4962165036752085e-05, "loss": 0.1271, "num_tokens": 641359501.0, "reward": 0.7243303954601288, "reward_std": 0.23969567194581032, "rewards/accuracy_reward/mean": 0.2366071380674839, "rewards/accuracy_reward/std": 0.3985053524374962, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.04931644396856427, "step": 1285 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 867.5201263427734, "completions/mean_terminated_length": 772.9378509521484, "completions/min_length": 334.25, "completions/min_terminated_length": 334.25, "epoch": 0.38413860055260995, "grad_norm": 0.2243078052997589, "kl": 2.8203125, "learning_rate": 1.4952691506439497e-05, "loss": 0.1481, "num_tokens": 641816470.0, "reward": 0.616629496216774, "reward_std": 0.12020007334649563, "rewards/accuracy_reward/mean": 0.13169642840512097, "rewards/accuracy_reward/std": 0.30193903110921383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05963814351707697, "step": 1286 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 897.6161041259766, "completions/mean_terminated_length": 814.7891082763672, "completions/min_length": 472.25, "completions/min_terminated_length": 472.25, "epoch": 0.3844373086401314, "grad_norm": 0.39447909593582153, "kl": 2.4677734375, "learning_rate": 1.4943212082861448e-05, "loss": 0.1292, "num_tokens": 642288842.0, "reward": 0.5870535895228386, "reward_std": 0.12717188894748688, "rewards/accuracy_reward/mean": 0.10044642724096775, "rewards/accuracy_reward/std": 0.2549739331007004, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.04669366031885147, "step": 1287 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 866.4419860839844, "completions/mean_terminated_length": 781.7625732421875, "completions/min_length": 437.25, "completions/min_terminated_length": 437.25, "epoch": 0.3847360167276529, "grad_norm": 0.18812140822410583, "kl": 2.193359375, "learning_rate": 1.4933726777297614e-05, "loss": 0.1158, "num_tokens": 642754064.0, "reward": 0.695870578289032, "reward_std": 0.14484280720353127, "rewards/accuracy_reward/mean": 0.2098214253783226, "rewards/accuracy_reward/std": 0.3368573784828186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.056536297313869, "step": 1288 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33035714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 850.9710235595703, "completions/mean_terminated_length": 767.4474029541016, "completions/min_length": 348.25, "completions/min_terminated_length": 348.25, "epoch": 0.38503472481517437, "grad_norm": 0.32538798451423645, "kl": 2.6171875, "learning_rate": 1.4924235601034673e-05, "loss": 0.1494, "num_tokens": 643207619.0, "reward": 0.5825892984867096, "reward_std": 0.12793445773422718, "rewards/accuracy_reward/mean": 0.09821428591385484, "rewards/accuracy_reward/std": 0.27182213217020035, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.060536280274391174, "step": 1289 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34821428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 854.1808319091797, "completions/mean_terminated_length": 762.8408966064453, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "epoch": 0.38533343290269584, "grad_norm": 0.46793094277381897, "kl": 2.83203125, "learning_rate": 1.4914738565366285e-05, "loss": 0.1404, "num_tokens": 643659140.0, "reward": 0.5312500149011612, "reward_std": 0.10977372759953141, "rewards/accuracy_reward/mean": 0.04687499930150807, "rewards/accuracy_reward/std": 0.16484765894711018, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.058870166540145874, "step": 1290 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 856.9174652099609, "completions/mean_terminated_length": 773.5751190185547, "completions/min_length": 445.75, "completions/min_terminated_length": 445.75, "epoch": 0.3856321409902173, "grad_norm": 0.1807224005460739, "kl": 2.474609375, "learning_rate": 1.4905235681593079e-05, "loss": 0.1402, "num_tokens": 644118271.0, "reward": 0.6199777126312256, "reward_std": 0.16315916925668716, "rewards/accuracy_reward/mean": 0.13392856810241938, "rewards/accuracy_reward/std": 0.30204131081700325, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05592204071581364, "step": 1291 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 862.4219055175781, "completions/mean_terminated_length": 773.6971588134766, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "epoch": 0.3859308490777388, "grad_norm": 0.2745739221572876, "kl": 2.6328125, "learning_rate": 1.4895726961022657e-05, "loss": 0.1443, "num_tokens": 644575516.0, "reward": 0.7020089477300644, "reward_std": 0.17351698875427246, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.4013185203075409, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.057508016005158424, "step": 1292 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36383928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 887.4353179931641, "completions/mean_terminated_length": 811.5384979248047, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.38622955716526025, "grad_norm": 0.43219876289367676, "kl": 2.35546875, "learning_rate": 1.4886212414969551e-05, "loss": 0.1119, "num_tokens": 645049423.0, "reward": 0.623325914144516, "reward_std": 0.17593121528625488, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.3385392166674137, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05868698377162218, "step": 1293 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33705357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 873.5826263427734, "completions/mean_terminated_length": 802.5887603759766, "completions/min_length": 460.5, "completions/min_terminated_length": 460.5, "epoch": 0.3865282652527817, "grad_norm": 0.3798691928386688, "kl": 1.763671875, "learning_rate": 1.4876692054755228e-05, "loss": 0.1059, "num_tokens": 645512820.0, "reward": 0.6389509290456772, "reward_std": 0.16986634209752083, "rewards/accuracy_reward/mean": 0.14955357206054032, "rewards/accuracy_reward/std": 0.3023831285536289, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 1294 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 866.7924499511719, "completions/mean_terminated_length": 751.2288970947266, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.3868269733403032, "grad_norm": 0.24823793768882751, "kl": 2.541015625, "learning_rate": 1.4867165891708082e-05, "loss": 0.1278, "num_tokens": 645980567.0, "reward": 0.5703125298023224, "reward_std": 0.12112263590097427, "rewards/accuracy_reward/mean": 0.08519345335662365, "rewards/accuracy_reward/std": 0.2275729440152645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05299014411866665, "step": 1295 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 852.7366485595703, "completions/mean_terminated_length": 768.5417938232422, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "epoch": 0.38712568142782466, "grad_norm": 0.24417978525161743, "kl": 2.3671875, "learning_rate": 1.4857633937163402e-05, "loss": 0.1226, "num_tokens": 646438705.0, "reward": 0.603236623108387, "reward_std": 0.1242286404594779, "rewards/accuracy_reward/mean": 0.1138392835855484, "rewards/accuracy_reward/std": 0.25755302608013153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 1296 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38169642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 865.919677734375, "completions/mean_terminated_length": 770.3425445556641, "completions/min_length": 393.25, "completions/min_terminated_length": 393.25, "epoch": 0.38742438951534613, "grad_norm": 0.27232927083969116, "kl": 1.64453125, "learning_rate": 1.4848096202463373e-05, "loss": 0.0912, "num_tokens": 646906365.0, "reward": 0.6501116305589676, "reward_std": 0.12588700838387012, "rewards/accuracy_reward/mean": 0.1562500037252903, "rewards/accuracy_reward/std": 0.3476349376142025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1297 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 920.1986999511719, "completions/mean_terminated_length": 819.5241241455078, "completions/min_length": 454.75, "completions/min_terminated_length": 454.75, "epoch": 0.3877230976028676, "grad_norm": 0.22910091280937195, "kl": 2.24609375, "learning_rate": 1.4838552698957054e-05, "loss": 0.1077, "num_tokens": 647399126.0, "reward": 0.6501116305589676, "reward_std": 0.13800288923084736, "rewards/accuracy_reward/mean": 0.16294642770662904, "rewards/accuracy_reward/std": 0.3237658962607384, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.055264041759073734, "step": 1298 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 910.794677734375, "completions/mean_terminated_length": 798.6492156982422, "completions/min_length": 445.5, "completions/min_terminated_length": 445.5, "epoch": 0.3880218056903891, "grad_norm": 0.4784952998161316, "kl": 1.533203125, "learning_rate": 1.4829003438000374e-05, "loss": 0.0721, "num_tokens": 647878634.0, "reward": 0.6462053954601288, "reward_std": 0.13228655280545354, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.3586985617876053, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 1299 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 901.185302734375, "completions/mean_terminated_length": 799.2595062255859, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "epoch": 0.38832051377791055, "grad_norm": 0.3232438564300537, "kl": 1.3525390625, "learning_rate": 1.4819448430956112e-05, "loss": 0.0645, "num_tokens": 648360989.0, "reward": 0.667410746216774, "reward_std": 0.16594024747610092, "rewards/accuracy_reward/mean": 0.17187500116415322, "rewards/accuracy_reward/std": 0.32630152627825737, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.027692769188433886, "step": 1300 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 885.8370819091797, "completions/mean_terminated_length": 765.7117004394531, "completions/min_length": 354.75, "completions/min_terminated_length": 354.75, "epoch": 0.388619221865432, "grad_norm": 0.19038031995296478, "kl": 1.1962890625, "learning_rate": 1.4809887689193878e-05, "loss": 0.0648, "num_tokens": 648822116.0, "reward": 0.7092634439468384, "reward_std": 0.16910918802022934, "rewards/accuracy_reward/mean": 0.2220982126891613, "rewards/accuracy_reward/std": 0.4155613109469414, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.030101283453404903, "step": 1301 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 919.8616638183594, "completions/mean_terminated_length": 784.6805572509766, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "epoch": 0.3889179299529535, "grad_norm": 0.18138107657432556, "kl": 1.005859375, "learning_rate": 1.4800321224090114e-05, "loss": 0.0528, "num_tokens": 649305526.0, "reward": 0.603794664144516, "reward_std": 0.12908046692609787, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.30514875426888466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 1302 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 864.7411041259766, "completions/mean_terminated_length": 768.3880004882812, "completions/min_length": 357.25, "completions/min_terminated_length": 357.25, "epoch": 0.38921663804047496, "grad_norm": 0.22030656039714813, "kl": 0.94921875, "learning_rate": 1.4790749047028065e-05, "loss": 0.0516, "num_tokens": 649763314.0, "reward": 0.620535746216774, "reward_std": 0.14555735886096954, "rewards/accuracy_reward/mean": 0.12500000093132257, "rewards/accuracy_reward/std": 0.3179612085223198, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1303 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 871.825927734375, "completions/mean_terminated_length": 759.4231567382812, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.38951534612799643, "grad_norm": 0.19374029338359833, "kl": 1.2197265625, "learning_rate": 1.4781171169397781e-05, "loss": 0.0665, "num_tokens": 650225316.0, "reward": 0.5820312798023224, "reward_std": 0.10866606421768665, "rewards/accuracy_reward/mean": 0.08928571199066937, "rewards/accuracy_reward/std": 0.25909905321896076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03955313144251704, "step": 1304 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 892.9710083007812, "completions/mean_terminated_length": 770.4430236816406, "completions/min_length": 363.75, "completions/min_terminated_length": 363.75, "epoch": 0.3898140542155179, "grad_norm": 0.166166290640831, "kl": 0.86328125, "learning_rate": 1.4771587602596085e-05, "loss": 0.0407, "num_tokens": 650693527.0, "reward": 0.6914062798023224, "reward_std": 0.1645062416791916, "rewards/accuracy_reward/mean": 0.1964285746216774, "rewards/accuracy_reward/std": 0.3185076639056206, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 1305 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 918.0379943847656, "completions/mean_terminated_length": 789.7853698730469, "completions/min_length": 407.5, "completions/min_terminated_length": 407.5, "epoch": 0.3901127623030394, "grad_norm": 0.14034393429756165, "kl": 0.734375, "learning_rate": 1.4761998358026581e-05, "loss": 0.036, "num_tokens": 651183928.0, "reward": 0.6992187798023224, "reward_std": 0.15509601309895515, "rewards/accuracy_reward/mean": 0.20312500465661287, "rewards/accuracy_reward/std": 0.37430205196142197, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.02676480822265148, "step": 1306 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 891.1696929931641, "completions/mean_terminated_length": 752.5597686767578, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "epoch": 0.39041147039056084, "grad_norm": 0.15665259957313538, "kl": 1.0595703125, "learning_rate": 1.4752403447099617e-05, "loss": 0.0675, "num_tokens": 651658244.0, "reward": 0.6266741454601288, "reward_std": 0.15895690396428108, "rewards/accuracy_reward/mean": 0.13169642817229033, "rewards/accuracy_reward/std": 0.31561552733182907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.029007501434534788, "step": 1307 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6830357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 970.935302734375, "completions/mean_terminated_length": 876.7868499755859, "completions/min_length": 548.5, "completions/min_terminated_length": 548.5, "epoch": 0.3907101784780823, "grad_norm": 0.1990279108285904, "kl": 1.033203125, "learning_rate": 1.4742802881232291e-05, "loss": 0.0451, "num_tokens": 652169719.0, "reward": 0.5747768133878708, "reward_std": 0.10671863332390785, "rewards/accuracy_reward/mean": 0.08035714155994356, "rewards/accuracy_reward/std": 0.23418232053518295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1308 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 920.0156555175781, "completions/mean_terminated_length": 792.1891174316406, "completions/min_length": 430.25, "completions/min_terminated_length": 430.25, "epoch": 0.3910088865656038, "grad_norm": 0.17765121161937714, "kl": 1.1259765625, "learning_rate": 1.4733196671848435e-05, "loss": 0.0629, "num_tokens": 652660910.0, "reward": 0.684709832072258, "reward_std": 0.1446425262838602, "rewards/accuracy_reward/mean": 0.19196428079158068, "rewards/accuracy_reward/std": 0.3643237054347992, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 1309 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6495535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 941.8549652099609, "completions/mean_terminated_length": 791.7347564697266, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "epoch": 0.39130759465312526, "grad_norm": 0.2304149717092514, "kl": 1.4501953125, "learning_rate": 1.4723584830378584e-05, "loss": 0.0764, "num_tokens": 653153197.0, "reward": 0.6149553656578064, "reward_std": 0.13996238075196743, "rewards/accuracy_reward/mean": 0.12500000093132257, "rewards/accuracy_reward/std": 0.3177771344780922, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.047033360693603754, "step": 1310 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 935.1763916015625, "completions/mean_terminated_length": 818.9864196777344, "completions/min_length": 490.75, "completions/min_terminated_length": 490.75, "epoch": 0.39160630274064673, "grad_norm": 0.26212379336357117, "kl": 1.0869140625, "learning_rate": 1.4713967368259981e-05, "loss": 0.0553, "num_tokens": 653644572.0, "reward": 0.6735491305589676, "reward_std": 0.17707857862114906, "rewards/accuracy_reward/mean": 0.18080356856808066, "rewards/accuracy_reward/std": 0.3469708114862442, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 1311 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6160714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 952.1339721679688, "completions/mean_terminated_length": 839.1026306152344, "completions/min_length": 531.0, "completions/min_terminated_length": 531.0, "epoch": 0.3919050108281682, "grad_norm": 0.19422854483127594, "kl": 1.0947265625, "learning_rate": 1.470434429693655e-05, "loss": 0.0529, "num_tokens": 654145800.0, "reward": 0.5691964477300644, "reward_std": 0.1075788983143866, "rewards/accuracy_reward/mean": 0.07812500093132257, "rewards/accuracy_reward/std": 0.21622870862483978, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589571990072727, "step": 1312 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 933.3638916015625, "completions/mean_terminated_length": 798.4214477539062, "completions/min_length": 486.75, "completions/min_terminated_length": 486.75, "epoch": 0.39220371891568967, "grad_norm": 0.29607370495796204, "kl": 1.2021484375, "learning_rate": 1.469471562785891e-05, "loss": 0.0618, "num_tokens": 654639275.0, "reward": 0.7098214477300644, "reward_std": 0.15995104424655437, "rewards/accuracy_reward/mean": 0.2187500037252903, "rewards/accuracy_reward/std": 0.3919530138373375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589571990072727, "step": 1313 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 894.529052734375, "completions/mean_terminated_length": 754.6855163574219, "completions/min_length": 369.5, "completions/min_terminated_length": 369.5, "epoch": 0.3925024270032111, "grad_norm": 0.24047403037548065, "kl": 1.681640625, "learning_rate": 1.4685081372484318e-05, "loss": 0.0799, "num_tokens": 655109560.0, "reward": 0.7014509290456772, "reward_std": 0.1921578124165535, "rewards/accuracy_reward/mean": 0.214285708963871, "rewards/accuracy_reward/std": 0.39296959340572357, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054504433646798134, "step": 1314 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6808035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 966.1719207763672, "completions/mean_terminated_length": 842.6281890869141, "completions/min_length": 474.75, "completions/min_terminated_length": 474.75, "epoch": 0.39280113509073256, "grad_norm": 0.18049627542495728, "kl": 1.751953125, "learning_rate": 1.4675441542276685e-05, "loss": 0.086, "num_tokens": 655610133.0, "reward": 0.7287946790456772, "reward_std": 0.21599692292511463, "rewards/accuracy_reward/mean": 0.2488839253783226, "rewards/accuracy_reward/std": 0.4317655488848686, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 1315 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 941.7701263427734, "completions/mean_terminated_length": 816.2056427001953, "completions/min_length": 426.5, "completions/min_terminated_length": 426.5, "epoch": 0.393099843178254, "grad_norm": 0.4266059100627899, "kl": 2.037109375, "learning_rate": 1.4665796148706561e-05, "loss": 0.0881, "num_tokens": 656096094.0, "reward": 0.6517857611179352, "reward_std": 0.17752252332866192, "rewards/accuracy_reward/mean": 0.1651785708963871, "rewards/accuracy_reward/std": 0.357146468013525, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05643500294536352, "step": 1316 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 929.4621124267578, "completions/mean_terminated_length": 790.6854095458984, "completions/min_length": 445.5, "completions/min_terminated_length": 445.5, "epoch": 0.3933985512657755, "grad_norm": 0.23443976044654846, "kl": 2.0234375, "learning_rate": 1.4656145203251116e-05, "loss": 0.0947, "num_tokens": 656601325.0, "reward": 0.564732164144516, "reward_std": 0.129008571151644, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.22562920674681664, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.055819165892899036, "step": 1317 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5870535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 936.8326263427734, "completions/mean_terminated_length": 809.7453765869141, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "epoch": 0.39369725935329697, "grad_norm": 0.29922711849212646, "kl": 1.98828125, "learning_rate": 1.4646488717394116e-05, "loss": 0.0949, "num_tokens": 657089538.0, "reward": 0.7031250447034836, "reward_std": 0.1380335185676813, "rewards/accuracy_reward/mean": 0.21428571082651615, "rewards/accuracy_reward/std": 0.3825792819261551, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 1318 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 951.9955749511719, "completions/mean_terminated_length": 845.1858978271484, "completions/min_length": 580.25, "completions/min_terminated_length": 580.25, "epoch": 0.39399596744081844, "grad_norm": 0.29205408692359924, "kl": 2.36328125, "learning_rate": 1.463682670262593e-05, "loss": 0.1122, "num_tokens": 657584208.0, "reward": 0.6395089626312256, "reward_std": 0.14450818672776222, "rewards/accuracy_reward/mean": 0.1569940517656505, "rewards/accuracy_reward/std": 0.32502079382538795, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06042194366455078, "step": 1319 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 948.4911193847656, "completions/mean_terminated_length": 843.5099639892578, "completions/min_length": 577.0, "completions/min_terminated_length": 577.0, "epoch": 0.3942946755283399, "grad_norm": 0.27121374011039734, "kl": 1.8828125, "learning_rate": 1.4627159170443504e-05, "loss": 0.0898, "num_tokens": 658078460.0, "reward": 0.6294643133878708, "reward_std": 0.1317103672772646, "rewards/accuracy_reward/mean": 0.14285714784637094, "rewards/accuracy_reward/std": 0.3015555143356323, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 1320 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6316964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 949.7232666015625, "completions/mean_terminated_length": 825.0387878417969, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "epoch": 0.3945933836158614, "grad_norm": 0.6032569408416748, "kl": 3.0234375, "learning_rate": 1.4617486132350343e-05, "loss": 0.1437, "num_tokens": 658580128.0, "reward": 0.5507812798023224, "reward_std": 0.144304933026433, "rewards/accuracy_reward/mean": 0.0714285725262016, "rewards/accuracy_reward/std": 0.2117837779223919, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526753783226, "rewards/tag_count_reward/std": 0.06852320581674576, "step": 1321 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 917.3080902099609, "completions/mean_terminated_length": 831.0744171142578, "completions/min_length": 514.25, "completions/min_terminated_length": 514.25, "epoch": 0.39489209170338285, "grad_norm": 0.2145966738462448, "kl": 1.359375, "learning_rate": 1.4607807599856507e-05, "loss": 0.0694, "num_tokens": 659070810.0, "reward": 0.5747768059372902, "reward_std": 0.13892407342791557, "rewards/accuracy_reward/mean": 0.08928571548312902, "rewards/accuracy_reward/std": 0.23272541910409927, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05698249954730272, "step": 1322 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 904.4732666015625, "completions/mean_terminated_length": 819.8317718505859, "completions/min_length": 507.5, "completions/min_terminated_length": 507.5, "epoch": 0.3951907997909043, "grad_norm": 0.27825894951820374, "kl": 1.94921875, "learning_rate": 1.45981235844786e-05, "loss": 0.1044, "num_tokens": 659548174.0, "reward": 0.5831473618745804, "reward_std": 0.1434993576258421, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.29037660360336304, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05817027762532234, "step": 1323 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 888.2232360839844, "completions/mean_terminated_length": 793.7220916748047, "completions/min_length": 408.75, "completions/min_terminated_length": 408.75, "epoch": 0.3954895078784258, "grad_norm": 0.36731138825416565, "kl": 1.7314453125, "learning_rate": 1.4588434097739744e-05, "loss": 0.1042, "num_tokens": 660014258.0, "reward": 0.5943080484867096, "reward_std": 0.13416343089193106, "rewards/accuracy_reward/mean": 0.10937500023283064, "rewards/accuracy_reward/std": 0.2719593159854412, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.057373433373868465, "step": 1324 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 898.3973541259766, "completions/mean_terminated_length": 818.515380859375, "completions/min_length": 487.0, "completions/min_terminated_length": 487.0, "epoch": 0.39578821596594727, "grad_norm": 0.22536896169185638, "kl": 1.91015625, "learning_rate": 1.4578739151169567e-05, "loss": 0.1055, "num_tokens": 660489972.0, "reward": 0.6032366305589676, "reward_std": 0.20028693415224552, "rewards/accuracy_reward/mean": 0.12276785634458065, "rewards/accuracy_reward/std": 0.31241776049137115, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 1325 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 872.029052734375, "completions/mean_terminated_length": 787.6703186035156, "completions/min_length": 459.0, "completions/min_terminated_length": 459.0, "epoch": 0.39608692405346874, "grad_norm": 0.3411339819431305, "kl": 2.037109375, "learning_rate": 1.4569038756304209e-05, "loss": 0.1139, "num_tokens": 660955329.0, "reward": 0.640066996216774, "reward_std": 0.1300586722791195, "rewards/accuracy_reward/mean": 0.1562500037252903, "rewards/accuracy_reward/std": 0.36023346334695816, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.05957930441945791, "step": 1326 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 848.9777069091797, "completions/mean_terminated_length": 784.3735046386719, "completions/min_length": 423.75, "completions/min_terminated_length": 423.75, "epoch": 0.3963856321409902, "grad_norm": 0.39224886894226074, "kl": 1.265625, "learning_rate": 1.4559332924686276e-05, "loss": 0.0815, "num_tokens": 661405095.0, "reward": 0.6523437798023224, "reward_std": 0.1661185324192047, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.3639083802700043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04288960574194789, "step": 1327 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23883928571428573, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 830.5156707763672, "completions/mean_terminated_length": 770.1182556152344, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.3966843402285117, "grad_norm": 0.2449573576450348, "kl": 1.6826171875, "learning_rate": 1.454962166786485e-05, "loss": 0.1049, "num_tokens": 661855806.0, "reward": 0.6523437798023224, "reward_std": 0.145844254642725, "rewards/accuracy_reward/mean": 0.16071428824216127, "rewards/accuracy_reward/std": 0.3464597463607788, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04334343643859029, "step": 1328 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2455357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 849.6317443847656, "completions/mean_terminated_length": 795.4943695068359, "completions/min_length": 426.5, "completions/min_terminated_length": 426.5, "epoch": 0.39698304831603315, "grad_norm": 0.182883620262146, "kl": 1.5869140625, "learning_rate": 1.4539904997395468e-05, "loss": 0.0773, "num_tokens": 662308921.0, "reward": 0.702566996216774, "reward_std": 0.16126768290996552, "rewards/accuracy_reward/mean": 0.2120535708963871, "rewards/accuracy_reward/std": 0.3909846320748329, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04537529917433858, "step": 1329 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.21205357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 822.0379943847656, "completions/mean_terminated_length": 770.1089782714844, "completions/min_length": 419.25, "completions/min_terminated_length": 419.25, "epoch": 0.3972817564035546, "grad_norm": 0.23630118370056152, "kl": 2.005859375, "learning_rate": 1.4530182924840117e-05, "loss": 0.1135, "num_tokens": 662748266.0, "reward": 0.6367187947034836, "reward_std": 0.17159250006079674, "rewards/accuracy_reward/mean": 0.14508928125724196, "rewards/accuracy_reward/std": 0.32556769251823425, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.044580988585948944, "step": 1330 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.22098214285714288, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 832.8482513427734, "completions/mean_terminated_length": 781.0046539306641, "completions/min_length": 416.0, "completions/min_terminated_length": 416.0, "epoch": 0.3975804644910761, "grad_norm": 0.24970345199108124, "kl": 1.8544921875, "learning_rate": 1.45204554617672e-05, "loss": 0.0919, "num_tokens": 663187606.0, "reward": 0.6205357313156128, "reward_std": 0.12904410809278488, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.33824336528778076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05112028680741787, "step": 1331 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.19642857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 794.1161193847656, "completions/mean_terminated_length": 738.0049438476562, "completions/min_length": 339.5, "completions/min_terminated_length": 339.5, "epoch": 0.39787917257859756, "grad_norm": 0.2802300751209259, "kl": 2.28125, "learning_rate": 1.4510722619751536e-05, "loss": 0.1285, "num_tokens": 663614874.0, "reward": 0.588169664144516, "reward_std": 0.18008005432784557, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.28965066373348236, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 1332 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 858.1920166015625, "completions/mean_terminated_length": 790.2201385498047, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.39817788066611903, "grad_norm": 0.16824455559253693, "kl": 1.876953125, "learning_rate": 1.4500984410374353e-05, "loss": 0.1041, "num_tokens": 664072256.0, "reward": 0.594866082072258, "reward_std": 0.1519713643938303, "rewards/accuracy_reward/mean": 0.10267857019789517, "rewards/accuracy_reward/std": 0.27624736353755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 1333 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.24330357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 828.1964569091797, "completions/mean_terminated_length": 765.8077392578125, "completions/min_length": 383.25, "completions/min_terminated_length": 383.25, "epoch": 0.3984765887536405, "grad_norm": 0.346866250038147, "kl": 2.248046875, "learning_rate": 1.4491240845223253e-05, "loss": 0.1321, "num_tokens": 664514344.0, "reward": 0.6300223469734192, "reward_std": 0.13457649946212769, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.3490893766283989, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055374542251229286, "step": 1334 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.19866071428571427, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 822.2277069091797, "completions/mean_terminated_length": 772.3242950439453, "completions/min_length": 398.5, "completions/min_terminated_length": 398.5, "epoch": 0.398775296841162, "grad_norm": 0.22175051271915436, "kl": 1.87109375, "learning_rate": 1.4481491935892227e-05, "loss": 0.09, "num_tokens": 664950078.0, "reward": 0.6277902126312256, "reward_std": 0.10553264804184437, "rewards/accuracy_reward/mean": 0.1361607122235, "rewards/accuracy_reward/std": 0.31641004979610443, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 1335 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2299107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 821.5469207763672, "completions/mean_terminated_length": 761.5739288330078, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "epoch": 0.39907400492868345, "grad_norm": 0.3821104168891907, "kl": 2.169921875, "learning_rate": 1.447173769398161e-05, "loss": 0.107, "num_tokens": 665389443.0, "reward": 0.6841517984867096, "reward_std": 0.15954296896234155, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.3925532400608063, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 1336 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33035714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 879.4018096923828, "completions/mean_terminated_length": 808.316650390625, "completions/min_length": 470.0, "completions/min_terminated_length": 470.0, "epoch": 0.3993727130162049, "grad_norm": 0.23027688264846802, "kl": 1.576171875, "learning_rate": 1.4461978131098089e-05, "loss": 0.0796, "num_tokens": 665854167.0, "reward": 0.6813616454601288, "reward_std": 0.18255885317921638, "rewards/accuracy_reward/mean": 0.18750000186264515, "rewards/accuracy_reward/std": 0.3746103122830391, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 1337 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 830.3839721679688, "completions/mean_terminated_length": 756.4834747314453, "completions/min_length": 422.5, "completions/min_terminated_length": 422.5, "epoch": 0.3996714211037264, "grad_norm": 0.21246875822544098, "kl": 2.06640625, "learning_rate": 1.4452213258854684e-05, "loss": 0.1136, "num_tokens": 666297443.0, "reward": 0.6199777126312256, "reward_std": 0.12514283880591393, "rewards/accuracy_reward/mean": 0.1339285746216774, "rewards/accuracy_reward/std": 0.27961114794015884, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 1338 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 863.9219055175781, "completions/mean_terminated_length": 792.4933166503906, "completions/min_length": 431.25, "completions/min_terminated_length": 431.25, "epoch": 0.39997012919124786, "grad_norm": 0.3357756733894348, "kl": 1.890625, "learning_rate": 1.4442443088870727e-05, "loss": 0.1029, "num_tokens": 666762048.0, "reward": 0.6021205484867096, "reward_std": 0.15188992209732533, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.31120144948363304, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.042059858329594135, "step": 1339 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 899.2254943847656, "completions/mean_terminated_length": 812.0115661621094, "completions/min_length": 433.0, "completions/min_terminated_length": 433.0, "epoch": 0.40026883727876933, "grad_norm": 0.21050679683685303, "kl": 2.09765625, "learning_rate": 1.443266763277186e-05, "loss": 0.1, "num_tokens": 667245413.0, "reward": 0.6372768133878708, "reward_std": 0.15608621202409267, "rewards/accuracy_reward/mean": 0.14955356903374195, "rewards/accuracy_reward/std": 0.34431688487529755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 1340 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 919.6920166015625, "completions/mean_terminated_length": 841.0127563476562, "completions/min_length": 505.0, "completions/min_terminated_length": 505.0, "epoch": 0.4005675453662908, "grad_norm": 0.26436489820480347, "kl": 1.216796875, "learning_rate": 1.4422886902190014e-05, "loss": 0.0624, "num_tokens": 667732075.0, "reward": 0.6590402126312256, "reward_std": 0.12580402195453644, "rewards/accuracy_reward/mean": 0.16517856810241938, "rewards/accuracy_reward/std": 0.3523806780576706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 1341 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 877.0781555175781, "completions/mean_terminated_length": 773.5792388916016, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 0.40086625345381227, "grad_norm": 0.1973269283771515, "kl": 1.078125, "learning_rate": 1.4413100908763391e-05, "loss": 0.0443, "num_tokens": 668186862.0, "reward": 0.6361607313156128, "reward_std": 0.12025694036856294, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.2795308195054531, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1342 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 894.8460083007812, "completions/mean_terminated_length": 757.4319305419922, "completions/min_length": 426.0, "completions/min_terminated_length": 426.0, "epoch": 0.40116496154133374, "grad_norm": 0.2983778119087219, "kl": 1.16015625, "learning_rate": 1.440330966413646e-05, "loss": 0.0571, "num_tokens": 668661849.0, "reward": 0.5848214477300644, "reward_std": 0.13049072958528996, "rewards/accuracy_reward/mean": 0.09151785867288709, "rewards/accuracy_reward/std": 0.26725272089242935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1343 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 944.3036193847656, "completions/mean_terminated_length": 845.9191436767578, "completions/min_length": 546.0, "completions/min_terminated_length": 546.0, "epoch": 0.4014636696288552, "grad_norm": 0.13292276859283447, "kl": 0.9775390625, "learning_rate": 1.4393513179959936e-05, "loss": 0.047, "num_tokens": 669154561.0, "reward": 0.6824777275323868, "reward_std": 0.08264908008277416, "rewards/accuracy_reward/mean": 0.18749999743886292, "rewards/accuracy_reward/std": 0.32431608252227306, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.494977667927742, "rewards/tag_count_reward/std": 0.03507901635020971, "step": 1344 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 918.4486999511719, "completions/mean_terminated_length": 813.5268707275391, "completions/min_length": 423.25, "completions/min_terminated_length": 423.25, "epoch": 0.4017623777163767, "grad_norm": 0.19786281883716583, "kl": 1.283203125, "learning_rate": 1.4383711467890776e-05, "loss": 0.0599, "num_tokens": 669635402.0, "reward": 0.6813616454601288, "reward_std": 0.1623614076524973, "rewards/accuracy_reward/mean": 0.18749999743886292, "rewards/accuracy_reward/std": 0.3449446987360716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.038415491580963135, "step": 1345 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 896.3326416015625, "completions/mean_terminated_length": 808.8755493164062, "completions/min_length": 456.5, "completions/min_terminated_length": 456.5, "epoch": 0.40206108580389815, "grad_norm": 0.1649295538663864, "kl": 1.03125, "learning_rate": 1.4373904539592145e-05, "loss": 0.0391, "num_tokens": 670112255.0, "reward": 0.5831473618745804, "reward_std": 0.1076370757073164, "rewards/accuracy_reward/mean": 0.08928571455180645, "rewards/accuracy_reward/std": 0.28060688078403473, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 1346 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 924.0670013427734, "completions/mean_terminated_length": 807.7110137939453, "completions/min_length": 481.5, "completions/min_terminated_length": 481.5, "epoch": 0.4023597938914196, "grad_norm": 0.20355500280857086, "kl": 1.087890625, "learning_rate": 1.436409240673342e-05, "loss": 0.0506, "num_tokens": 670595197.0, "reward": 0.5809152126312256, "reward_std": 0.08061248250305653, "rewards/accuracy_reward/mean": 0.08705357136204839, "rewards/accuracy_reward/std": 0.2624290883541107, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1347 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 912.5111999511719, "completions/mean_terminated_length": 759.483642578125, "completions/min_length": 414.25, "completions/min_terminated_length": 414.25, "epoch": 0.4026585019789411, "grad_norm": 0.24705903232097626, "kl": 1.06689453125, "learning_rate": 1.435427508099018e-05, "loss": 0.0552, "num_tokens": 671078850.0, "reward": 0.6138393133878708, "reward_std": 0.13285165769048035, "rewards/accuracy_reward/mean": 0.12053571734577417, "rewards/accuracy_reward/std": 0.23853980749845505, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.033647436648607254, "step": 1348 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 914.3460235595703, "completions/mean_terminated_length": 772.6619110107422, "completions/min_length": 357.75, "completions/min_terminated_length": 357.75, "epoch": 0.40295721006646257, "grad_norm": 0.37442582845687866, "kl": 1.5859375, "learning_rate": 1.4344452574044173e-05, "loss": 0.0879, "num_tokens": 671558813.0, "reward": 0.6199777126312256, "reward_std": 0.20477572456002235, "rewards/accuracy_reward/mean": 0.13169642724096775, "rewards/accuracy_reward/std": 0.32997073978185654, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05209200643002987, "step": 1349 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 914.8906707763672, "completions/mean_terminated_length": 799.3218383789062, "completions/min_length": 435.75, "completions/min_terminated_length": 435.75, "epoch": 0.40325591815398404, "grad_norm": 0.1679074913263321, "kl": 1.287109375, "learning_rate": 1.4334624897583308e-05, "loss": 0.0554, "num_tokens": 672034636.0, "reward": 0.6902902126312256, "reward_std": 0.1374283730983734, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.37447943538427353, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03258697595447302, "step": 1350 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 938.9330749511719, "completions/mean_terminated_length": 818.5934295654297, "completions/min_length": 397.25, "completions/min_terminated_length": 397.25, "epoch": 0.4035546262415055, "grad_norm": 0.1835741102695465, "kl": 1.4482421875, "learning_rate": 1.4324792063301662e-05, "loss": 0.0695, "num_tokens": 672531246.0, "reward": 0.6350446790456772, "reward_std": 0.17164032813161612, "rewards/accuracy_reward/mean": 0.14285714481957257, "rewards/accuracy_reward/std": 0.2949548065662384, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 1351 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 897.1986999511719, "completions/mean_terminated_length": 784.3669128417969, "completions/min_length": 293.25, "completions/min_terminated_length": 293.25, "epoch": 0.403853334329027, "grad_norm": 0.1650969684123993, "kl": 1.5, "learning_rate": 1.4314954082899435e-05, "loss": 0.0663, "num_tokens": 673005687.0, "reward": 0.6132812798023224, "reward_std": 0.14340505748987198, "rewards/accuracy_reward/mean": 0.12276785541325808, "rewards/accuracy_reward/std": 0.30794814974069595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 1352 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 924.2388916015625, "completions/mean_terminated_length": 803.1826934814453, "completions/min_length": 460.75, "completions/min_terminated_length": 460.75, "epoch": 0.40415204241654845, "grad_norm": 0.3896518051624298, "kl": 1.6884765625, "learning_rate": 1.4305110968082953e-05, "loss": 0.0886, "num_tokens": 673494882.0, "reward": 0.6880580633878708, "reward_std": 0.1960228644311428, "rewards/accuracy_reward/mean": 0.19642857275903225, "rewards/accuracy_reward/std": 0.3912675678730011, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.043475935235619545, "step": 1353 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 935.8839721679688, "completions/mean_terminated_length": 844.7794189453125, "completions/min_length": 435.5, "completions/min_terminated_length": 435.5, "epoch": 0.4044507505040699, "grad_norm": 0.2532058656215668, "kl": 1.3115234375, "learning_rate": 1.4295262730564654e-05, "loss": 0.0683, "num_tokens": 673990158.0, "reward": 0.7081473618745804, "reward_std": 0.1958363577723503, "rewards/accuracy_reward/mean": 0.214285708963871, "rewards/accuracy_reward/std": 0.4005560055375099, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 1354 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 886.8460235595703, "completions/mean_terminated_length": 787.7763671875, "completions/min_length": 400.75, "completions/min_terminated_length": 400.75, "epoch": 0.4047494585915914, "grad_norm": 0.23410357534885406, "kl": 2.140625, "learning_rate": 1.4285409382063074e-05, "loss": 0.1119, "num_tokens": 674462249.0, "reward": 0.5993303954601288, "reward_std": 0.14514479041099548, "rewards/accuracy_reward/mean": 0.11160714458674192, "rewards/accuracy_reward/std": 0.23934891819953918, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05369503889232874, "step": 1355 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 927.0268249511719, "completions/mean_terminated_length": 799.9292297363281, "completions/min_length": 432.75, "completions/min_terminated_length": 432.75, "epoch": 0.40504816667911286, "grad_norm": 0.25961410999298096, "kl": 1.8994140625, "learning_rate": 1.4275550934302822e-05, "loss": 0.1094, "num_tokens": 674956581.0, "reward": 0.6316964626312256, "reward_std": 0.1999354474246502, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.34455253183841705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 1356 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 932.2879943847656, "completions/mean_terminated_length": 814.6842651367188, "completions/min_length": 518.5, "completions/min_terminated_length": 518.5, "epoch": 0.4053468747666343, "grad_norm": 0.35300612449645996, "kl": 1.3076171875, "learning_rate": 1.4265687399014584e-05, "loss": 0.07, "num_tokens": 675443766.0, "reward": 0.647879496216774, "reward_std": 0.1427355408668518, "rewards/accuracy_reward/mean": 0.15885416604578495, "rewards/accuracy_reward/std": 0.3493855744600296, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.042492654640227556, "step": 1357 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38616071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 848.966552734375, "completions/mean_terminated_length": 745.745849609375, "completions/min_length": 323.5, "completions/min_terminated_length": 323.5, "epoch": 0.40564558285415575, "grad_norm": 0.20319779217243195, "kl": 1.953125, "learning_rate": 1.4255818787935097e-05, "loss": 0.0994, "num_tokens": 675893591.0, "reward": 0.5837053954601288, "reward_std": 0.16154582053422928, "rewards/accuracy_reward/mean": 0.09858630783855915, "rewards/accuracy_reward/std": 0.285240039229393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053606295958161354, "step": 1358 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 893.0848846435547, "completions/mean_terminated_length": 793.3541870117188, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.4059442909416772, "grad_norm": 0.18268951773643494, "kl": 1.46875, "learning_rate": 1.4245945112807133e-05, "loss": 0.0735, "num_tokens": 676365501.0, "reward": 0.705357164144516, "reward_std": 0.15678295399993658, "rewards/accuracy_reward/mean": 0.2142857164144516, "rewards/accuracy_reward/std": 0.40556421875953674, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04355311533436179, "step": 1359 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 950.9687805175781, "completions/mean_terminated_length": 860.0904083251953, "completions/min_length": 443.75, "completions/min_terminated_length": 443.75, "epoch": 0.4062429990291987, "grad_norm": 0.2931804955005646, "kl": 1.564453125, "learning_rate": 1.4236066385379497e-05, "loss": 0.0687, "num_tokens": 676857407.0, "reward": 0.5770089626312256, "reward_std": 0.14192447252571583, "rewards/accuracy_reward/mean": 0.08705357182770967, "rewards/accuracy_reward/std": 0.2742365263402462, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048092021606862545, "step": 1360 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 917.5714569091797, "completions/mean_terminated_length": 774.09814453125, "completions/min_length": 364.5, "completions/min_terminated_length": 364.5, "epoch": 0.40654170711672016, "grad_norm": 0.22799883782863617, "kl": 1.947265625, "learning_rate": 1.4226182617406996e-05, "loss": 0.0942, "num_tokens": 677335807.0, "reward": 0.6590402126312256, "reward_std": 0.16808640025556087, "rewards/accuracy_reward/mean": 0.17187500139698386, "rewards/accuracy_reward/std": 0.3442055583000183, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055120271630585194, "step": 1361 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 914.1919860839844, "completions/mean_terminated_length": 805.0729675292969, "completions/min_length": 439.5, "completions/min_terminated_length": 439.5, "epoch": 0.40684041520424163, "grad_norm": 0.20149344205856323, "kl": 1.921875, "learning_rate": 1.4216293820650447e-05, "loss": 0.0958, "num_tokens": 677814005.0, "reward": 0.6339286118745804, "reward_std": 0.1641868744045496, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.3408683277666569, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392761349678, "rewards/tag_count_reward/std": 0.050723335705697536, "step": 1362 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 904.4152221679688, "completions/mean_terminated_length": 777.7509765625, "completions/min_length": 348.75, "completions/min_terminated_length": 348.75, "epoch": 0.4071391232917631, "grad_norm": 0.174734964966774, "kl": 1.66748046875, "learning_rate": 1.4206400006876644e-05, "loss": 0.0794, "num_tokens": 678290271.0, "reward": 0.6623884290456772, "reward_std": 0.12023353017866611, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.3645358458161354, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.045718629378825426, "step": 1363 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 919.1630096435547, "completions/mean_terminated_length": 810.3796081542969, "completions/min_length": 417.75, "completions/min_terminated_length": 417.75, "epoch": 0.4074378313792846, "grad_norm": 0.3319390118122101, "kl": 2.3515625, "learning_rate": 1.4196501187858346e-05, "loss": 0.1075, "num_tokens": 678767448.0, "reward": 0.6406250298023224, "reward_std": 0.20313630998134613, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.35990428924560547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.06053628120571375, "step": 1364 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49553571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 904.2946929931641, "completions/mean_terminated_length": 790.6125030517578, "completions/min_length": 427.75, "completions/min_terminated_length": 427.75, "epoch": 0.40773653946680605, "grad_norm": 0.26094141602516174, "kl": 2.439453125, "learning_rate": 1.4186597375374283e-05, "loss": 0.1249, "num_tokens": 679252332.0, "reward": 0.6339285969734192, "reward_std": 0.17226684093475342, "rewards/accuracy_reward/mean": 0.14955356903374195, "rewards/accuracy_reward/std": 0.32776231318712234, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05994668882340193, "step": 1365 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5892857142857144, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 947.2522888183594, "completions/mean_terminated_length": 837.3964233398438, "completions/min_length": 432.5, "completions/min_terminated_length": 432.5, "epoch": 0.4080352475543275, "grad_norm": 0.20846548676490784, "kl": 2.34765625, "learning_rate": 1.4176688581209109e-05, "loss": 0.1045, "num_tokens": 679749517.0, "reward": 0.7215402126312256, "reward_std": 0.22953956201672554, "rewards/accuracy_reward/mean": 0.23883928544819355, "rewards/accuracy_reward/std": 0.3907340168952942, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.0626716511324048, "step": 1366 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 922.0045013427734, "completions/mean_terminated_length": 821.9611968994141, "completions/min_length": 490.0, "completions/min_terminated_length": 490.0, "epoch": 0.408333955641849, "grad_norm": 0.2661169171333313, "kl": 2.30859375, "learning_rate": 1.416677481715342e-05, "loss": 0.1095, "num_tokens": 680234463.0, "reward": 0.5781250298023224, "reward_std": 0.15181208960711956, "rewards/accuracy_reward/mean": 0.09375000232830644, "rewards/accuracy_reward/std": 0.2717808596789837, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.06008276715874672, "step": 1367 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 904.3236999511719, "completions/mean_terminated_length": 773.6867980957031, "completions/min_length": 304.5, "completions/min_terminated_length": 304.5, "epoch": 0.40863266372937046, "grad_norm": 0.27299758791923523, "kl": 2.33984375, "learning_rate": 1.415685609500371e-05, "loss": 0.1131, "num_tokens": 680718432.0, "reward": 0.7438616454601288, "reward_std": 0.2044351939111948, "rewards/accuracy_reward/mean": 0.2589285746216774, "rewards/accuracy_reward/std": 0.39235997945070267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05963814351707697, "step": 1368 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 911.1607666015625, "completions/mean_terminated_length": 772.5059814453125, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "epoch": 0.40893137181689193, "grad_norm": 0.2958131432533264, "kl": 2.318359375, "learning_rate": 1.4146932426562391e-05, "loss": 0.1194, "num_tokens": 681195992.0, "reward": 0.5742187798023224, "reward_std": 0.1375578111037612, "rewards/accuracy_reward/mean": 0.08928571455180645, "rewards/accuracy_reward/std": 0.2391808032989502, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05943890102207661, "step": 1369 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 886.0714721679688, "completions/mean_terminated_length": 782.9247283935547, "completions/min_length": 422.5, "completions/min_terminated_length": 422.5, "epoch": 0.4092300799044134, "grad_norm": 0.7304915189743042, "kl": 2.22265625, "learning_rate": 1.4137003823637753e-05, "loss": 0.1309, "num_tokens": 681662040.0, "reward": 0.678013414144516, "reward_std": 0.24181564897298813, "rewards/accuracy_reward/mean": 0.21465773321688175, "rewards/accuracy_reward/std": 0.3841356858611107, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4748883992433548, "rewards/tag_count_reward/std": 0.07303918432444334, "step": 1370 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 924.0446929931641, "completions/mean_terminated_length": 822.3194885253906, "completions/min_length": 281.75, "completions/min_terminated_length": 281.75, "epoch": 0.4095287879919349, "grad_norm": 0.36589112877845764, "kl": 1.5869140625, "learning_rate": 1.4127070298043949e-05, "loss": 0.0705, "num_tokens": 682148892.0, "reward": 0.5708705633878708, "reward_std": 0.13374876091256738, "rewards/accuracy_reward/mean": 0.08258928544819355, "rewards/accuracy_reward/std": 0.22714244201779366, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.04255996737629175, "step": 1371 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 856.0826416015625, "completions/mean_terminated_length": 770.5454559326172, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.40982749607945634, "grad_norm": 0.22766569256782532, "kl": 2.154296875, "learning_rate": 1.4117131861601003e-05, "loss": 0.1087, "num_tokens": 682603569.0, "reward": 0.6077009290456772, "reward_std": 0.17709668539464474, "rewards/accuracy_reward/mean": 0.1205357126891613, "rewards/accuracy_reward/std": 0.3048676326870918, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05450443457812071, "step": 1372 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 895.7277374267578, "completions/mean_terminated_length": 748.924072265625, "completions/min_length": 268.75, "completions/min_terminated_length": 268.75, "epoch": 0.4101262041669778, "grad_norm": 0.29675939679145813, "kl": 1.97265625, "learning_rate": 1.4107188526134774e-05, "loss": 0.1029, "num_tokens": 683074439.0, "reward": 0.6344866305589676, "reward_std": 0.14484448079019785, "rewards/accuracy_reward/mean": 0.14508928172290325, "rewards/accuracy_reward/std": 0.2840779945254326, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 1373 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44866071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 884.2522735595703, "completions/mean_terminated_length": 775.1158752441406, "completions/min_length": 289.25, "completions/min_terminated_length": 289.25, "epoch": 0.4104249122544993, "grad_norm": 0.25400641560554504, "kl": 2.1591796875, "learning_rate": 1.4097240303476955e-05, "loss": 0.1038, "num_tokens": 683543720.0, "reward": 0.6277902126312256, "reward_std": 0.19854821637272835, "rewards/accuracy_reward/mean": 0.14062500186264515, "rewards/accuracy_reward/std": 0.3373427093029022, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.051970279309898615, "step": 1374 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 892.3683624267578, "completions/mean_terminated_length": 789.8265228271484, "completions/min_length": 237.0, "completions/min_terminated_length": 237.0, "epoch": 0.41072362034202076, "grad_norm": 0.2414836436510086, "kl": 2.158203125, "learning_rate": 1.408728720546505e-05, "loss": 0.1087, "num_tokens": 684020989.0, "reward": 0.5597098469734192, "reward_std": 0.12607308849692345, "rewards/accuracy_reward/mean": 0.07142857322469354, "rewards/accuracy_reward/std": 0.24126021191477776, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 1375 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 891.9263763427734, "completions/mean_terminated_length": 796.9125671386719, "completions/min_length": 328.25, "completions/min_terminated_length": 328.25, "epoch": 0.4110223284295422, "grad_norm": 0.24180804193019867, "kl": 2.005859375, "learning_rate": 1.4077329243942368e-05, "loss": 0.1069, "num_tokens": 684492972.0, "reward": 0.640066996216774, "reward_std": 0.15617859177291393, "rewards/accuracy_reward/mean": 0.1495535671710968, "rewards/accuracy_reward/std": 0.3322540447115898, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 1376 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 821.0402221679688, "completions/mean_terminated_length": 725.1113739013672, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "epoch": 0.4113210365170637, "grad_norm": 0.45911651849746704, "kl": 1.833984375, "learning_rate": 1.4067366430758004e-05, "loss": 0.086, "num_tokens": 684933150.0, "reward": 0.6618303805589676, "reward_std": 0.13077778927981853, "rewards/accuracy_reward/mean": 0.1741071413271129, "rewards/accuracy_reward/std": 0.34814631938934326, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03161557484418154, "step": 1377 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 865.5067291259766, "completions/mean_terminated_length": 758.3721160888672, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "epoch": 0.41161974460458517, "grad_norm": 0.3891010880470276, "kl": 2.26953125, "learning_rate": 1.4057398777766824e-05, "loss": 0.1204, "num_tokens": 685391969.0, "reward": 0.647879496216774, "reward_std": 0.22216356173157692, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.35665713250637054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 1378 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 907.575927734375, "completions/mean_terminated_length": 810.4195251464844, "completions/min_length": 393.0, "completions/min_terminated_length": 393.0, "epoch": 0.41191845269210664, "grad_norm": 0.42910176515579224, "kl": 1.794921875, "learning_rate": 1.4047426296829455e-05, "loss": 0.0853, "num_tokens": 685874595.0, "reward": 0.6997768133878708, "reward_std": 0.21751971915364265, "rewards/accuracy_reward/mean": 0.20758928218856454, "rewards/accuracy_reward/std": 0.36688829585909843, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 1379 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 855.7857360839844, "completions/mean_terminated_length": 779.5263977050781, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "epoch": 0.4122171607796281, "grad_norm": 0.21542397141456604, "kl": 1.6943359375, "learning_rate": 1.4037448999812272e-05, "loss": 0.092, "num_tokens": 686337475.0, "reward": 0.6138393208384514, "reward_std": 0.12294571241363883, "rewards/accuracy_reward/mean": 0.12276785634458065, "rewards/accuracy_reward/std": 0.26490428298711777, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044403897132724524, "step": 1380 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31026785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 820.1652069091797, "completions/mean_terminated_length": 727.3975219726562, "completions/min_length": 281.75, "completions/min_terminated_length": 281.75, "epoch": 0.4125158688671496, "grad_norm": 0.24467432498931885, "kl": 2.005859375, "learning_rate": 1.4027466898587375e-05, "loss": 0.1099, "num_tokens": 686772413.0, "reward": 0.7020089626312256, "reward_std": 0.15489895083010197, "rewards/accuracy_reward/mean": 0.2120535671710968, "rewards/accuracy_reward/std": 0.4058016315102577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 1381 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 902.763427734375, "completions/mean_terminated_length": 794.3728942871094, "completions/min_length": 230.75, "completions/min_terminated_length": 230.75, "epoch": 0.41281457695467105, "grad_norm": 0.17821575701236725, "kl": 1.111328125, "learning_rate": 1.4017480005032578e-05, "loss": 0.0515, "num_tokens": 687245171.0, "reward": 0.6300223469734192, "reward_std": 0.16299950890243053, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.33067718893289566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521269638091326, "step": 1382 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 889.9419860839844, "completions/mean_terminated_length": 766.9219512939453, "completions/min_length": 239.25, "completions/min_terminated_length": 239.25, "epoch": 0.4131132850421925, "grad_norm": 0.2214488834142685, "kl": 1.3154296875, "learning_rate": 1.4007488331031409e-05, "loss": 0.0581, "num_tokens": 687713769.0, "reward": 0.6093750298023224, "reward_std": 0.15861237701028585, "rewards/accuracy_reward/mean": 0.11830357415601611, "rewards/accuracy_reward/std": 0.2843465469777584, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03817511722445488, "step": 1383 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 854.6183319091797, "completions/mean_terminated_length": 731.4077301025391, "completions/min_length": 239.0, "completions/min_terminated_length": 239.0, "epoch": 0.413411993129714, "grad_norm": 0.26725319027900696, "kl": 0.9609375, "learning_rate": 1.3997491888473079e-05, "loss": 0.0557, "num_tokens": 688178398.0, "reward": 0.6361607313156128, "reward_std": 0.08940745377913117, "rewards/accuracy_reward/mean": 0.1406250037252903, "rewards/accuracy_reward/std": 0.2868475914001465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1384 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 903.5424499511719, "completions/mean_terminated_length": 790.10546875, "completions/min_length": 292.25, "completions/min_terminated_length": 292.25, "epoch": 0.41371070121723547, "grad_norm": 0.15385453402996063, "kl": 1.341796875, "learning_rate": 1.3987490689252463e-05, "loss": 0.0681, "num_tokens": 688657729.0, "reward": 0.6495535969734192, "reward_std": 0.1733321901410818, "rewards/accuracy_reward/mean": 0.1584821413271129, "rewards/accuracy_reward/std": 0.34155331552028656, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714402794838, "rewards/tag_count_reward/std": 0.04480193881317973, "step": 1385 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 826.841552734375, "completions/mean_terminated_length": 727.871826171875, "completions/min_length": 288.5, "completions/min_terminated_length": 288.5, "epoch": 0.41400940930475694, "grad_norm": 0.219356507062912, "kl": 1.287109375, "learning_rate": 1.3977484745270112e-05, "loss": 0.0751, "num_tokens": 689098618.0, "reward": 0.5970982313156128, "reward_std": 0.16284853033721447, "rewards/accuracy_reward/mean": 0.10491071594879031, "rewards/accuracy_reward/std": 0.28833482414484024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 1386 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 915.6183319091797, "completions/mean_terminated_length": 822.87451171875, "completions/min_length": 481.75, "completions/min_terminated_length": 481.75, "epoch": 0.4143081173922784, "grad_norm": 0.30024081468582153, "kl": 0.84326171875, "learning_rate": 1.3967474068432212e-05, "loss": 0.0429, "num_tokens": 689581503.0, "reward": 0.6082589477300644, "reward_std": 0.1585737895220518, "rewards/accuracy_reward/mean": 0.11383928637951612, "rewards/accuracy_reward/std": 0.3071446865797043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 1387 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 892.1652221679688, "completions/mean_terminated_length": 777.6557769775391, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "epoch": 0.4146068254797999, "grad_norm": 0.2488042414188385, "kl": 1.4052734375, "learning_rate": 1.3957458670650587e-05, "loss": 0.0828, "num_tokens": 690053865.0, "reward": 0.7399553954601288, "reward_std": 0.1947278007864952, "rewards/accuracy_reward/mean": 0.2477678544819355, "rewards/accuracy_reward/std": 0.4138660356402397, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 1388 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 894.8415679931641, "completions/mean_terminated_length": 771.5572357177734, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 0.41490553356732135, "grad_norm": 0.3092111647129059, "kl": 1.14111328125, "learning_rate": 1.3947438563842672e-05, "loss": 0.0684, "num_tokens": 690526626.0, "reward": 0.6138393133878708, "reward_std": 0.13153945468366146, "rewards/accuracy_reward/mean": 0.1205357164144516, "rewards/accuracy_reward/std": 0.31402868404984474, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.034245037473738194, "step": 1389 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 858.3527069091797, "completions/mean_terminated_length": 743.102294921875, "completions/min_length": 195.75, "completions/min_terminated_length": 195.75, "epoch": 0.4152042416548428, "grad_norm": 0.16944973170757294, "kl": 2.07421875, "learning_rate": 1.3937413759931515e-05, "loss": 0.1166, "num_tokens": 690977072.0, "reward": 0.607700914144516, "reward_std": 0.08813472324982285, "rewards/accuracy_reward/mean": 0.11830356903374195, "rewards/accuracy_reward/std": 0.2727055773139, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04696295503526926, "step": 1390 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 874.9174499511719, "completions/mean_terminated_length": 769.6879425048828, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.4155029497423643, "grad_norm": 0.2377190738916397, "kl": 1.703125, "learning_rate": 1.3927384270845744e-05, "loss": 0.0885, "num_tokens": 691438235.0, "reward": 0.7109375298023224, "reward_std": 0.09682843554764986, "rewards/accuracy_reward/mean": 0.2209821455180645, "rewards/accuracy_reward/std": 0.4110952913761139, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 1391 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 871.2746124267578, "completions/mean_terminated_length": 758.7606658935547, "completions/min_length": 287.5, "completions/min_terminated_length": 287.5, "epoch": 0.41580165782988576, "grad_norm": 0.2776210606098175, "kl": 2.4140625, "learning_rate": 1.391735010851956e-05, "loss": 0.1307, "num_tokens": 691911334.0, "reward": 0.674107164144516, "reward_std": 0.2187163233757019, "rewards/accuracy_reward/mean": 0.1875000037252903, "rewards/accuracy_reward/std": 0.380135640501976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05536533612757921, "step": 1392 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 869.4085235595703, "completions/mean_terminated_length": 764.3576507568359, "completions/min_length": 440.5, "completions/min_terminated_length": 440.5, "epoch": 0.41610036591740723, "grad_norm": 0.20419831573963165, "kl": 1.474609375, "learning_rate": 1.3907311284892737e-05, "loss": 0.0732, "num_tokens": 692380397.0, "reward": 0.686941996216774, "reward_std": 0.15664693526923656, "rewards/accuracy_reward/mean": 0.20275297574698925, "rewards/accuracy_reward/std": 0.37963613495230675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 1393 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 923.1295013427734, "completions/mean_terminated_length": 823.8933715820312, "completions/min_length": 483.75, "completions/min_terminated_length": 483.75, "epoch": 0.4163990740049287, "grad_norm": 0.3273962140083313, "kl": 2.15625, "learning_rate": 1.3897267811910589e-05, "loss": 0.101, "num_tokens": 692869383.0, "reward": 0.6802455633878708, "reward_std": 0.18067756667733192, "rewards/accuracy_reward/mean": 0.19196428637951612, "rewards/accuracy_reward/std": 0.37661221623420715, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918200105429, "step": 1394 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 919.7812805175781, "completions/mean_terminated_length": 789.9300689697266, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "epoch": 0.4166977820924502, "grad_norm": 0.33523181080818176, "kl": 2.302734375, "learning_rate": 1.3887219701523958e-05, "loss": 0.1259, "num_tokens": 693354485.0, "reward": 0.656808078289032, "reward_std": 0.20664040371775627, "rewards/accuracy_reward/mean": 0.16964286006987095, "rewards/accuracy_reward/std": 0.3577134720981121, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487165167927742, "rewards/tag_count_reward/std": 0.05365365277975798, "step": 1395 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 938.1920166015625, "completions/mean_terminated_length": 829.1221466064453, "completions/min_length": 482.0, "completions/min_terminated_length": 482.0, "epoch": 0.41699649017997165, "grad_norm": 0.4307641088962555, "kl": 2.2265625, "learning_rate": 1.3877166965689206e-05, "loss": 0.1048, "num_tokens": 693846107.0, "reward": 0.6506696492433548, "reward_std": 0.19191743060946465, "rewards/accuracy_reward/mean": 0.1629464286379516, "rewards/accuracy_reward/std": 0.3439597934484482, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 1396 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 885.8839721679688, "completions/mean_terminated_length": 757.5029754638672, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "epoch": 0.4172951982674931, "grad_norm": 0.32960245013237, "kl": 2.146484375, "learning_rate": 1.3867109616368208e-05, "loss": 0.1184, "num_tokens": 694317463.0, "reward": 0.5987723469734192, "reward_std": 0.10225777048617601, "rewards/accuracy_reward/mean": 0.11160714086145163, "rewards/accuracy_reward/std": 0.25207582116127014, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05466675665229559, "step": 1397 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 919.7812805175781, "completions/mean_terminated_length": 795.9809417724609, "completions/min_length": 442.25, "completions/min_terminated_length": 442.25, "epoch": 0.4175939063550146, "grad_norm": 0.2482239305973053, "kl": 2.107421875, "learning_rate": 1.3857047665528317e-05, "loss": 0.0942, "num_tokens": 694800965.0, "reward": 0.5770089626312256, "reward_std": 0.11699424171820283, "rewards/accuracy_reward/mean": 0.09151785727590322, "rewards/accuracy_reward/std": 0.2327229231595993, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05674629285931587, "step": 1398 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 896.8482666015625, "completions/mean_terminated_length": 770.6102905273438, "completions/min_length": 465.5, "completions/min_terminated_length": 465.5, "epoch": 0.41789261444253606, "grad_norm": 0.1695907562971115, "kl": 1.712890625, "learning_rate": 1.3846981125142363e-05, "loss": 0.0891, "num_tokens": 695270657.0, "reward": 0.6406250298023224, "reward_std": 0.141687773168087, "rewards/accuracy_reward/mean": 0.15178571362048388, "rewards/accuracy_reward/std": 0.33464596420526505, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05035856366157532, "step": 1399 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 877.1540679931641, "completions/mean_terminated_length": 771.0600128173828, "completions/min_length": 370.25, "completions/min_terminated_length": 370.25, "epoch": 0.4181913225300575, "grad_norm": 0.29556718468666077, "kl": 1.8623046875, "learning_rate": 1.3836910007188642e-05, "loss": 0.104, "num_tokens": 695733590.0, "reward": 0.6339285969734192, "reward_std": 0.12780625745654106, "rewards/accuracy_reward/mean": 0.1473214253783226, "rewards/accuracy_reward/std": 0.34965961426496506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05252540344372392, "step": 1400 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 903.2433471679688, "completions/mean_terminated_length": 745.1609649658203, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "epoch": 0.41849003061757895, "grad_norm": 0.3940150737762451, "kl": 1.53125, "learning_rate": 1.3826834323650899e-05, "loss": 0.0796, "num_tokens": 696211651.0, "reward": 0.5731027126312256, "reward_std": 0.1214490607380867, "rewards/accuracy_reward/mean": 0.08482142956927419, "rewards/accuracy_reward/std": 0.2489483430981636, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 1401 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 907.0290679931641, "completions/mean_terminated_length": 797.2846221923828, "completions/min_length": 500.0, "completions/min_terminated_length": 500.0, "epoch": 0.4187887387051004, "grad_norm": 0.23983041942119598, "kl": 2.1015625, "learning_rate": 1.3816754086518305e-05, "loss": 0.1019, "num_tokens": 696689216.0, "reward": 0.6311384290456772, "reward_std": 0.17623435705900192, "rewards/accuracy_reward/mean": 0.14508928591385484, "rewards/accuracy_reward/std": 0.32547036185860634, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.05626536812633276, "step": 1402 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 908.1920013427734, "completions/mean_terminated_length": 813.6796722412109, "completions/min_length": 486.75, "completions/min_terminated_length": 486.75, "epoch": 0.4190874467926219, "grad_norm": 0.5748719573020935, "kl": 1.677734375, "learning_rate": 1.3806669307785447e-05, "loss": 0.0897, "num_tokens": 697177398.0, "reward": 0.6300223544239998, "reward_std": 0.11857862398028374, "rewards/accuracy_reward/mean": 0.1428571455180645, "rewards/accuracy_reward/std": 0.28907906264066696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05331833194941282, "step": 1403 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 912.1540679931641, "completions/mean_terminated_length": 800.2628021240234, "completions/min_length": 481.5, "completions/min_terminated_length": 481.5, "epoch": 0.41938615488014336, "grad_norm": 0.2412521094083786, "kl": 1.9462890625, "learning_rate": 1.3796579999452328e-05, "loss": 0.1057, "num_tokens": 697658203.0, "reward": 0.7148437798023224, "reward_std": 0.25263458862900734, "rewards/accuracy_reward/mean": 0.2276785708963871, "rewards/accuracy_reward/std": 0.4174683764576912, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05233129486441612, "step": 1404 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 908.2500457763672, "completions/mean_terminated_length": 799.11181640625, "completions/min_length": 491.0, "completions/min_terminated_length": 491.0, "epoch": 0.41968486296766483, "grad_norm": 0.23487624526023865, "kl": 2.5390625, "learning_rate": 1.3786486173524331e-05, "loss": 0.1289, "num_tokens": 698144747.0, "reward": 0.6640625298023224, "reward_std": 0.2132595181465149, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.3663114458322525, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06243238039314747, "step": 1405 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 901.3661041259766, "completions/mean_terminated_length": 772.0470275878906, "completions/min_length": 418.75, "completions/min_terminated_length": 418.75, "epoch": 0.4199835710551863, "grad_norm": 0.29070836305618286, "kl": 2.5, "learning_rate": 1.3776387842012217e-05, "loss": 0.1269, "num_tokens": 698615855.0, "reward": 0.6093750298023224, "reward_std": 0.16394312120974064, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.3338286802172661, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.479910708963871, "rewards/tag_count_reward/std": 0.06778926029801369, "step": 1406 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 908.2879943847656, "completions/mean_terminated_length": 782.3909454345703, "completions/min_length": 503.75, "completions/min_terminated_length": 503.75, "epoch": 0.42028227914270777, "grad_norm": 0.19919613003730774, "kl": 2.421875, "learning_rate": 1.3766285016932109e-05, "loss": 0.121, "num_tokens": 699095552.0, "reward": 0.619977705180645, "reward_std": 0.14490927942097187, "rewards/accuracy_reward/mean": 0.13616071385331452, "rewards/accuracy_reward/std": 0.31020849756896496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.060639469884335995, "step": 1407 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 905.7656555175781, "completions/mean_terminated_length": 792.1062316894531, "completions/min_length": 498.0, "completions/min_terminated_length": 498.0, "epoch": 0.42058098723022924, "grad_norm": 0.2885380983352661, "kl": 2.634765625, "learning_rate": 1.3756177710305476e-05, "loss": 0.1298, "num_tokens": 699587719.0, "reward": 0.643415205180645, "reward_std": 0.1743279634974897, "rewards/accuracy_reward/mean": 0.1607142798602581, "rewards/accuracy_reward/std": 0.2285623401403427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.061792174354195595, "step": 1408 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 892.4241485595703, "completions/mean_terminated_length": 788.0758972167969, "completions/min_length": 363.5, "completions/min_terminated_length": 363.5, "epoch": 0.4208796953177507, "grad_norm": 0.5576410889625549, "kl": 3.642578125, "learning_rate": 1.3746065934159123e-05, "loss": 0.1701, "num_tokens": 700060613.0, "reward": 0.5859375298023224, "reward_std": 0.2400386743247509, "rewards/accuracy_reward/mean": 0.10714285541325808, "rewards/accuracy_reward/std": 0.3007071688771248, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.06738371960818768, "step": 1409 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 887.0625305175781, "completions/mean_terminated_length": 781.9157104492188, "completions/min_length": 443.5, "completions/min_terminated_length": 443.5, "epoch": 0.4211784034052722, "grad_norm": 0.33167600631713867, "kl": 4.04296875, "learning_rate": 1.3735949700525164e-05, "loss": 0.2177, "num_tokens": 700528113.0, "reward": 0.564174123108387, "reward_std": 0.1264092894271016, "rewards/accuracy_reward/mean": 0.08705357206054032, "rewards/accuracy_reward/std": 0.2117458563297987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.477120541036129, "rewards/tag_count_reward/std": 0.07217255048453808, "step": 1410 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 867.5781555175781, "completions/mean_terminated_length": 761.7318878173828, "completions/min_length": 305.25, "completions/min_terminated_length": 305.25, "epoch": 0.42147711149279365, "grad_norm": 0.2812853455543518, "kl": 2.48046875, "learning_rate": 1.372582902144103e-05, "loss": 0.1381, "num_tokens": 700990948.0, "reward": 0.6043527126312256, "reward_std": 0.20502851530909538, "rewards/accuracy_reward/mean": 0.12053571175783873, "rewards/accuracy_reward/std": 0.2990358807146549, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06117267720401287, "step": 1411 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 886.4308319091797, "completions/mean_terminated_length": 762.4055023193359, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "epoch": 0.4217758195803151, "grad_norm": 0.3276362717151642, "kl": 3.25, "learning_rate": 1.3715703908949434e-05, "loss": 0.1703, "num_tokens": 701470309.0, "reward": 0.702566996216774, "reward_std": 0.2415020540356636, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.41208622604608536, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526828289032, "rewards/tag_count_reward/std": 0.06892330013215542, "step": 1412 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 860.3326110839844, "completions/mean_terminated_length": 753.0987854003906, "completions/min_length": 386.5, "completions/min_terminated_length": 386.5, "epoch": 0.4220745276678366, "grad_norm": 0.2847706377506256, "kl": 1.775390625, "learning_rate": 1.3705574375098365e-05, "loss": 0.088, "num_tokens": 701934938.0, "reward": 0.7773437947034836, "reward_std": 0.2023823969066143, "rewards/accuracy_reward/mean": 0.2857142873108387, "rewards/accuracy_reward/std": 0.4401969909667969, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 1413 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 827.1027069091797, "completions/mean_terminated_length": 750.2324981689453, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "epoch": 0.42237323575535807, "grad_norm": 0.3572338819503784, "kl": 1.484375, "learning_rate": 1.3695440431941069e-05, "loss": 0.0761, "num_tokens": 702382392.0, "reward": 0.6796875298023224, "reward_std": 0.19884305633604527, "rewards/accuracy_reward/mean": 0.18973214039579034, "rewards/accuracy_reward/std": 0.3367214910686016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 1414 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.25, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 789.8147735595703, "completions/mean_terminated_length": 714.82763671875, "completions/min_length": 345.25, "completions/min_terminated_length": 345.25, "epoch": 0.42267194384287954, "grad_norm": 0.24332354962825775, "kl": 2.00390625, "learning_rate": 1.3685302091536052e-05, "loss": 0.1031, "num_tokens": 702806645.0, "reward": 0.6450893133878708, "reward_std": 0.1779723595827818, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.3526618145406246, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 1415 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 826.154052734375, "completions/mean_terminated_length": 715.5485229492188, "completions/min_length": 358.75, "completions/min_terminated_length": 358.75, "epoch": 0.422970651930401, "grad_norm": 0.39689934253692627, "kl": 1.857421875, "learning_rate": 1.3675159365947038e-05, "loss": 0.1072, "num_tokens": 703249690.0, "reward": 0.6261161118745804, "reward_std": 0.12941107898950577, "rewards/accuracy_reward/mean": 0.15066964249126613, "rewards/accuracy_reward/std": 0.2798254173249006, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 1416 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 857.3616485595703, "completions/mean_terminated_length": 746.4357299804688, "completions/min_length": 383.5, "completions/min_terminated_length": 383.5, "epoch": 0.4232693600179225, "grad_norm": 0.1836618036031723, "kl": 1.8291015625, "learning_rate": 1.3665012267242974e-05, "loss": 0.1001, "num_tokens": 703709916.0, "reward": 0.616629496216774, "reward_std": 0.14656459726393223, "rewards/accuracy_reward/mean": 0.125000003259629, "rewards/accuracy_reward/std": 0.2887180335819721, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 1417 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3258928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 857.2277069091797, "completions/mean_terminated_length": 776.2774505615234, "completions/min_length": 305.25, "completions/min_terminated_length": 305.25, "epoch": 0.42356806810544395, "grad_norm": 0.45855259895324707, "kl": 1.943359375, "learning_rate": 1.3654860807498014e-05, "loss": 0.1085, "num_tokens": 704163250.0, "reward": 0.6629464626312256, "reward_std": 0.15424913354218006, "rewards/accuracy_reward/mean": 0.1741071380674839, "rewards/accuracy_reward/std": 0.36323710158467293, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051120287738740444, "step": 1418 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 834.3080749511719, "completions/mean_terminated_length": 752.0813903808594, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "epoch": 0.4238667761929654, "grad_norm": 0.48816460371017456, "kl": 1.521484375, "learning_rate": 1.3644704998791501e-05, "loss": 0.0757, "num_tokens": 704618028.0, "reward": 0.6830357313156128, "reward_std": 0.19878707826137543, "rewards/accuracy_reward/mean": 0.18750000186264515, "rewards/accuracy_reward/std": 0.3648752272129059, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.031776280142366886, "step": 1419 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 809.4263763427734, "completions/mean_terminated_length": 725.5948791503906, "completions/min_length": 350.25, "completions/min_terminated_length": 350.25, "epoch": 0.4241654842804869, "grad_norm": 0.2770076394081116, "kl": 2.05078125, "learning_rate": 1.3634544853207943e-05, "loss": 0.1038, "num_tokens": 705055979.0, "reward": 0.599888414144516, "reward_std": 0.13709265738725662, "rewards/accuracy_reward/mean": 0.1112351194024086, "rewards/accuracy_reward/std": 0.3132997080683708, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03606722131371498, "step": 1420 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 842.4821929931641, "completions/mean_terminated_length": 774.6085357666016, "completions/min_length": 468.25, "completions/min_terminated_length": 468.25, "epoch": 0.42446419236800836, "grad_norm": 0.20491845905780792, "kl": 1.421875, "learning_rate": 1.3624380382837017e-05, "loss": 0.0715, "num_tokens": 705505747.0, "reward": 0.6411830484867096, "reward_std": 0.10730361752212048, "rewards/accuracy_reward/mean": 0.14732142770662904, "rewards/accuracy_reward/std": 0.3166404478251934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03293030522763729, "step": 1421 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 823.6540679931641, "completions/mean_terminated_length": 741.1240539550781, "completions/min_length": 361.5, "completions/min_terminated_length": 361.5, "epoch": 0.42476290045552983, "grad_norm": 0.14120282232761383, "kl": 1.10400390625, "learning_rate": 1.3614211599773553e-05, "loss": 0.0699, "num_tokens": 705947512.0, "reward": 0.7656250447034836, "reward_std": 0.19957608357071877, "rewards/accuracy_reward/mean": 0.2700892835855484, "rewards/accuracy_reward/std": 0.4401380270719528, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 1422 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29464285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 844.5692291259766, "completions/mean_terminated_length": 773.9199371337891, "completions/min_length": 356.75, "completions/min_terminated_length": 356.75, "epoch": 0.4250616085430513, "grad_norm": 0.14919842779636383, "kl": 1.3896484375, "learning_rate": 1.36040385161175e-05, "loss": 0.0697, "num_tokens": 706401927.0, "reward": 0.6545759290456772, "reward_std": 0.144551953766495, "rewards/accuracy_reward/mean": 0.1607142873108387, "rewards/accuracy_reward/std": 0.2979578897356987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616156578064, "rewards/tag_count_reward/std": 0.036427486687898636, "step": 1423 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33258928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 848.8326263427734, "completions/mean_terminated_length": 763.6701202392578, "completions/min_length": 400.5, "completions/min_terminated_length": 400.5, "epoch": 0.4253603166305728, "grad_norm": 0.22154918313026428, "kl": 0.958984375, "learning_rate": 1.3593861143973922e-05, "loss": 0.0433, "num_tokens": 706862892.0, "reward": 0.5993303805589676, "reward_std": 0.10142829129472375, "rewards/accuracy_reward/mean": 0.10267857136204839, "rewards/accuracy_reward/std": 0.21521202102303505, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 1424 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 850.7790679931641, "completions/mean_terminated_length": 748.5340270996094, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "epoch": 0.42565902471809425, "grad_norm": 0.18403229117393494, "kl": 1.3388671875, "learning_rate": 1.3583679495453e-05, "loss": 0.0708, "num_tokens": 707315337.0, "reward": 0.6629464477300644, "reward_std": 0.1549163144081831, "rewards/accuracy_reward/mean": 0.16964286006987095, "rewards/accuracy_reward/std": 0.3635095953941345, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1425 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 845.6964569091797, "completions/mean_terminated_length": 749.3126373291016, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.4259577328056157, "grad_norm": 0.15252791345119476, "kl": 0.84521484375, "learning_rate": 1.3573493582670003e-05, "loss": 0.0386, "num_tokens": 707774593.0, "reward": 0.6171875149011612, "reward_std": 0.11954897455871105, "rewards/accuracy_reward/mean": 0.1205357126891613, "rewards/accuracy_reward/std": 0.3263646364212036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02843980584293604, "step": 1426 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 861.6897735595703, "completions/mean_terminated_length": 756.7391662597656, "completions/min_length": 350.75, "completions/min_terminated_length": 350.75, "epoch": 0.4262564408931372, "grad_norm": 0.3163803815841675, "kl": 1.0087890625, "learning_rate": 1.3563303417745258e-05, "loss": 0.0582, "num_tokens": 708229958.0, "reward": 0.79073666036129, "reward_std": 0.24462129175662994, "rewards/accuracy_reward/mean": 0.2968749962747097, "rewards/accuracy_reward/std": 0.4345351308584213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1427 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 900.8861999511719, "completions/mean_terminated_length": 777.2917633056641, "completions/min_length": 356.5, "completions/min_terminated_length": 356.5, "epoch": 0.42655514898065866, "grad_norm": 0.15348339080810547, "kl": 1.23388671875, "learning_rate": 1.3553109012804162e-05, "loss": 0.0569, "num_tokens": 708702627.0, "reward": 0.6702009290456772, "reward_std": 0.18400271236896515, "rewards/accuracy_reward/mean": 0.17633928451687098, "rewards/accuracy_reward/std": 0.3552696704864502, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03293030522763729, "step": 1428 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 892.0223541259766, "completions/mean_terminated_length": 792.9665679931641, "completions/min_length": 506.25, "completions/min_terminated_length": 506.25, "epoch": 0.42685385706818013, "grad_norm": 0.35610371828079224, "kl": 0.8564453125, "learning_rate": 1.3542910379977158e-05, "loss": 0.0569, "num_tokens": 709170493.0, "reward": 0.742745578289032, "reward_std": 0.16501018218696117, "rewards/accuracy_reward/mean": 0.2477678544819355, "rewards/accuracy_reward/std": 0.4168435037136078, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.033598463982343674, "step": 1429 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 881.7232666015625, "completions/mean_terminated_length": 770.5990142822266, "completions/min_length": 426.5, "completions/min_terminated_length": 426.5, "epoch": 0.4271525651557016, "grad_norm": 0.17255130410194397, "kl": 0.647216796875, "learning_rate": 1.353270753139972e-05, "loss": 0.0446, "num_tokens": 709629649.0, "reward": 0.7126116305589676, "reward_std": 0.12426945380866528, "rewards/accuracy_reward/mean": 0.214285708963871, "rewards/accuracy_reward/std": 0.40492308884859085, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258917927742, "rewards/tag_count_reward/std": 0.010136391967535019, "step": 1430 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 897.4955749511719, "completions/mean_terminated_length": 792.9734954833984, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.4274512732432231, "grad_norm": 0.3112136423587799, "kl": 1.6181640625, "learning_rate": 1.3522500479212337e-05, "loss": 0.0807, "num_tokens": 710101951.0, "reward": 0.5987723618745804, "reward_std": 0.13287124410271645, "rewards/accuracy_reward/mean": 0.10714285564608872, "rewards/accuracy_reward/std": 0.2844301201403141, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 1431 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 875.419677734375, "completions/mean_terminated_length": 788.7099609375, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "epoch": 0.42774998133074454, "grad_norm": 0.3412891924381256, "kl": 1.00146484375, "learning_rate": 1.3512289235560494e-05, "loss": 0.0479, "num_tokens": 710571627.0, "reward": 0.647879496216774, "reward_std": 0.16382554918527603, "rewards/accuracy_reward/mean": 0.15178571920841932, "rewards/accuracy_reward/std": 0.3377794995903969, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 1432 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 898.8928833007812, "completions/mean_terminated_length": 794.9358673095703, "completions/min_length": 453.0, "completions/min_terminated_length": 453.0, "epoch": 0.428048689418266, "grad_norm": 0.17773355543613434, "kl": 1.48828125, "learning_rate": 1.3502073812594677e-05, "loss": 0.0738, "num_tokens": 711048027.0, "reward": 0.6015625298023224, "reward_std": 0.12328251264989376, "rewards/accuracy_reward/mean": 0.11049107322469354, "rewards/accuracy_reward/std": 0.2764444537460804, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04272336792200804, "step": 1433 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 906.5736846923828, "completions/mean_terminated_length": 774.7794952392578, "completions/min_length": 406.5, "completions/min_terminated_length": 406.5, "epoch": 0.4283473975057875, "grad_norm": 0.22198174893856049, "kl": 1.1806640625, "learning_rate": 1.3491854222470332e-05, "loss": 0.0586, "num_tokens": 711530252.0, "reward": 0.7053571790456772, "reward_std": 0.1650850884616375, "rewards/accuracy_reward/mean": 0.2098214328289032, "rewards/accuracy_reward/std": 0.3858543522655964, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.031776280142366886, "step": 1434 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 884.3705749511719, "completions/mean_terminated_length": 758.6412200927734, "completions/min_length": 440.75, "completions/min_terminated_length": 440.75, "epoch": 0.42864610559330896, "grad_norm": 0.24684396386146545, "kl": 0.99560546875, "learning_rate": 1.3481630477347864e-05, "loss": 0.0453, "num_tokens": 711996098.0, "reward": 0.631138414144516, "reward_std": 0.1425110027194023, "rewards/accuracy_reward/mean": 0.13616071501746774, "rewards/accuracy_reward/std": 0.31916024163365364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 1435 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 895.8906707763672, "completions/mean_terminated_length": 794.1926727294922, "completions/min_length": 448.75, "completions/min_terminated_length": 448.75, "epoch": 0.42894481368083043, "grad_norm": 0.5610162615776062, "kl": 2.158203125, "learning_rate": 1.347140258939264e-05, "loss": 0.1064, "num_tokens": 712469313.0, "reward": 0.650669664144516, "reward_std": 0.1512417420744896, "rewards/accuracy_reward/mean": 0.15848214738070965, "rewards/accuracy_reward/std": 0.36316925287246704, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04357414972037077, "step": 1436 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 909.3326416015625, "completions/mean_terminated_length": 813.4915924072266, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.4292435217683519, "grad_norm": 0.2003922164440155, "kl": 1.19140625, "learning_rate": 1.346117057077493e-05, "loss": 0.0648, "num_tokens": 712943750.0, "reward": 0.6316964626312256, "reward_std": 0.1407323305029422, "rewards/accuracy_reward/mean": 0.1361607164144516, "rewards/accuracy_reward/std": 0.2804109752178192, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357238650322, "rewards/tag_count_reward/std": 0.030682499054819345, "step": 1437 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 907.9643249511719, "completions/mean_terminated_length": 778.9270629882812, "completions/min_length": 450.25, "completions/min_terminated_length": 450.25, "epoch": 0.42954222985587337, "grad_norm": 0.1459210067987442, "kl": 1.6279296875, "learning_rate": 1.3450934433669942e-05, "loss": 0.0795, "num_tokens": 713422598.0, "reward": 0.6445312649011612, "reward_std": 0.10349974781274796, "rewards/accuracy_reward/mean": 0.1517857164144516, "rewards/accuracy_reward/std": 0.29626789689064026, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040314854588359594, "step": 1438 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 927.029052734375, "completions/mean_terminated_length": 793.0331115722656, "completions/min_length": 428.75, "completions/min_terminated_length": 428.75, "epoch": 0.42984093794339484, "grad_norm": 0.18098533153533936, "kl": 1.689453125, "learning_rate": 1.3440694190257768e-05, "loss": 0.0843, "num_tokens": 713909027.0, "reward": 0.5719866305589676, "reward_std": 0.12051583081483841, "rewards/accuracy_reward/mean": 0.08035714365541935, "rewards/accuracy_reward/std": 0.2307768613100052, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04508844017982483, "step": 1439 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 896.5335388183594, "completions/mean_terminated_length": 771.7544403076172, "completions/min_length": 368.5, "completions/min_terminated_length": 368.5, "epoch": 0.4301396460309163, "grad_norm": 0.15731914341449738, "kl": 1.349609375, "learning_rate": 1.34304498527234e-05, "loss": 0.0556, "num_tokens": 714387970.0, "reward": 0.6160714626312256, "reward_std": 0.09213025122880936, "rewards/accuracy_reward/mean": 0.12500000186264515, "rewards/accuracy_reward/std": 0.3092636726796627, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03713445644825697, "step": 1440 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 935.5357666015625, "completions/mean_terminated_length": 827.9577026367188, "completions/min_length": 547.0, "completions/min_terminated_length": 547.0, "epoch": 0.4304383541184378, "grad_norm": 0.4173577129840851, "kl": 1.666015625, "learning_rate": 1.342020143325669e-05, "loss": 0.0826, "num_tokens": 714878578.0, "reward": 0.5401785895228386, "reward_std": 0.061724668368697166, "rewards/accuracy_reward/mean": 0.05691964365541935, "rewards/accuracy_reward/std": 0.1559237614274025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 1441 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 880.5803985595703, "completions/mean_terminated_length": 748.278564453125, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "epoch": 0.43073706220595925, "grad_norm": 0.1694755256175995, "kl": 1.453125, "learning_rate": 1.3409948944052352e-05, "loss": 0.0791, "num_tokens": 715345462.0, "reward": 0.6735491454601288, "reward_std": 0.14679337851703167, "rewards/accuracy_reward/mean": 0.18303571455180645, "rewards/accuracy_reward/std": 0.3835267499089241, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1442 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 960.5223846435547, "completions/mean_terminated_length": 855.3412475585938, "completions/min_length": 583.0, "completions/min_terminated_length": 583.0, "epoch": 0.43103577029348067, "grad_norm": 0.2661912143230438, "kl": 1.2138671875, "learning_rate": 1.3399692397309943e-05, "loss": 0.059, "num_tokens": 715841728.0, "reward": 0.6143973469734192, "reward_std": 0.13600238226354122, "rewards/accuracy_reward/mean": 0.12276786146685481, "rewards/accuracy_reward/std": 0.28221726045012474, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 1443 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 904.9241638183594, "completions/mean_terminated_length": 772.8641510009766, "completions/min_length": 431.25, "completions/min_terminated_length": 431.25, "epoch": 0.43133447838100214, "grad_norm": 0.19317476451396942, "kl": 1.716796875, "learning_rate": 1.3389431805233852e-05, "loss": 0.0904, "num_tokens": 716326190.0, "reward": 0.6026785895228386, "reward_std": 0.1505021397024393, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.2393336072564125, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051463617011904716, "step": 1444 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 917.8504791259766, "completions/mean_terminated_length": 784.7381286621094, "completions/min_length": 403.25, "completions/min_terminated_length": 403.25, "epoch": 0.4316331864685236, "grad_norm": 0.18015635013580322, "kl": 1.7666015625, "learning_rate": 1.3379167180033272e-05, "loss": 0.0848, "num_tokens": 716809115.0, "reward": 0.5926339626312256, "reward_std": 0.10004793968982995, "rewards/accuracy_reward/mean": 0.10491071827709675, "rewards/accuracy_reward/std": 0.23691707104444504, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.050699302926659584, "step": 1445 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 902.0491485595703, "completions/mean_terminated_length": 765.3795928955078, "completions/min_length": 418.5, "completions/min_terminated_length": 418.5, "epoch": 0.4319318945560451, "grad_norm": 0.23478993773460388, "kl": 1.74609375, "learning_rate": 1.3368898533922202e-05, "loss": 0.0871, "num_tokens": 717297697.0, "reward": 0.6830357611179352, "reward_std": 0.16526369377970695, "rewards/accuracy_reward/mean": 0.19419643026776612, "rewards/accuracy_reward/std": 0.34465859457850456, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 1446 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6495535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 936.1451263427734, "completions/mean_terminated_length": 784.510009765625, "completions/min_length": 332.75, "completions/min_terminated_length": 332.75, "epoch": 0.43223060264356655, "grad_norm": 0.17218098044395447, "kl": 1.69140625, "learning_rate": 1.3358625879119424e-05, "loss": 0.0865, "num_tokens": 717786962.0, "reward": 0.6819196939468384, "reward_std": 0.17390093952417374, "rewards/accuracy_reward/mean": 0.19196428265422583, "rewards/accuracy_reward/std": 0.3627338334918022, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 1447 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 906.966552734375, "completions/mean_terminated_length": 772.7196807861328, "completions/min_length": 439.5, "completions/min_terminated_length": 439.5, "epoch": 0.432529310731088, "grad_norm": 0.2795732319355011, "kl": 2.046875, "learning_rate": 1.3348349227848495e-05, "loss": 0.1014, "num_tokens": 718277091.0, "reward": 0.6752232313156128, "reward_std": 0.13673501648008823, "rewards/accuracy_reward/mean": 0.18749999743886292, "rewards/accuracy_reward/std": 0.3421649467200041, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 1448 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 896.6607360839844, "completions/mean_terminated_length": 749.2609252929688, "completions/min_length": 411.25, "completions/min_terminated_length": 411.25, "epoch": 0.4328280188186095, "grad_norm": 0.2807060778141022, "kl": 2.064453125, "learning_rate": 1.333806859233771e-05, "loss": 0.1044, "num_tokens": 718752395.0, "reward": 0.717075914144516, "reward_std": 0.12870756164193153, "rewards/accuracy_reward/mean": 0.2299107164144516, "rewards/accuracy_reward/std": 0.41070812195539474, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05405060388147831, "step": 1449 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6584821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 956.7612152099609, "completions/mean_terminated_length": 835.6058197021484, "completions/min_length": 486.25, "completions/min_terminated_length": 486.25, "epoch": 0.43312672690613097, "grad_norm": 0.3875960409641266, "kl": 2.03173828125, "learning_rate": 1.332778398482013e-05, "loss": 0.0889, "num_tokens": 719248336.0, "reward": 0.6406250298023224, "reward_std": 0.16893024742603302, "rewards/accuracy_reward/mean": 0.15401786006987095, "rewards/accuracy_reward/std": 0.34185653179883957, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.0531412404961884, "step": 1450 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6383928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 939.6875457763672, "completions/mean_terminated_length": 797.4762115478516, "completions/min_length": 520.0, "completions/min_terminated_length": 520.0, "epoch": 0.43342543499365244, "grad_norm": 0.2379845529794693, "kl": 3.015625, "learning_rate": 1.3317495417533523e-05, "loss": 0.1488, "num_tokens": 719739188.0, "reward": 0.649553582072258, "reward_std": 0.1540986504405737, "rewards/accuracy_reward/mean": 0.1674107122235, "rewards/accuracy_reward/std": 0.33562420308589935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06396691780537367, "step": 1451 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 903.0201263427734, "completions/mean_terminated_length": 785.2814025878906, "completions/min_length": 396.25, "completions/min_terminated_length": 396.25, "epoch": 0.4337241430811739, "grad_norm": 0.2049790769815445, "kl": 2.02734375, "learning_rate": 1.3307202902720377e-05, "loss": 0.1035, "num_tokens": 720214765.0, "reward": 0.666294664144516, "reward_std": 0.12833867967128754, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.38877827674150467, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05248269159346819, "step": 1452 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6339285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 941.2187957763672, "completions/mean_terminated_length": 806.42529296875, "completions/min_length": 445.5, "completions/min_terminated_length": 445.5, "epoch": 0.4340228511686954, "grad_norm": 0.23957429826259613, "kl": 2.15625, "learning_rate": 1.3296906452627874e-05, "loss": 0.096, "num_tokens": 720709919.0, "reward": 0.529575914144516, "reward_std": 0.08966807322576642, "rewards/accuracy_reward/mean": 0.04017857322469354, "rewards/accuracy_reward/std": 0.13512051105499268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04875553119927645, "step": 1453 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 942.1295166015625, "completions/mean_terminated_length": 814.5296173095703, "completions/min_length": 539.25, "completions/min_terminated_length": 539.25, "epoch": 0.43432155925621685, "grad_norm": 0.22127185761928558, "kl": 2.255859375, "learning_rate": 1.3286606079507881e-05, "loss": 0.1054, "num_tokens": 721201929.0, "reward": 0.6261161044239998, "reward_std": 0.15039020869880915, "rewards/accuracy_reward/mean": 0.14062500465661287, "rewards/accuracy_reward/std": 0.2624177113175392, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.057508016005158424, "step": 1454 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 886.1161193847656, "completions/mean_terminated_length": 768.1181335449219, "completions/min_length": 391.5, "completions/min_terminated_length": 391.5, "epoch": 0.4346202673437383, "grad_norm": 0.19126397371292114, "kl": 1.900390625, "learning_rate": 1.3276301795616937e-05, "loss": 0.0973, "num_tokens": 721667261.0, "reward": 0.6540178805589676, "reward_std": 0.15773649141192436, "rewards/accuracy_reward/mean": 0.16517856949940324, "rewards/accuracy_reward/std": 0.3340810053050518, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051717888563871384, "step": 1455 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 921.2478179931641, "completions/mean_terminated_length": 800.5196990966797, "completions/min_length": 459.5, "completions/min_terminated_length": 459.5, "epoch": 0.4349189754312598, "grad_norm": 0.28294646739959717, "kl": 2.70703125, "learning_rate": 1.3265993613216223e-05, "loss": 0.1323, "num_tokens": 722147788.0, "reward": 0.6774553805589676, "reward_std": 0.1929440125823021, "rewards/accuracy_reward/mean": 0.19419642724096775, "rewards/accuracy_reward/std": 0.37848827987909317, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483258917927742, "rewards/tag_count_reward/std": 0.062406137585639954, "step": 1456 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 947.122802734375, "completions/mean_terminated_length": 848.3447418212891, "completions/min_length": 466.25, "completions/min_terminated_length": 466.25, "epoch": 0.43521768351878126, "grad_norm": 0.2915401756763458, "kl": 2.404296875, "learning_rate": 1.3255681544571568e-05, "loss": 0.1174, "num_tokens": 722636435.0, "reward": 0.6568080633878708, "reward_std": 0.20205239206552505, "rewards/accuracy_reward/mean": 0.1741071455180645, "rewards/accuracy_reward/std": 0.3783346489071846, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06130507308989763, "step": 1457 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 934.8080749511719, "completions/mean_terminated_length": 812.5658874511719, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "epoch": 0.43551639160630273, "grad_norm": 0.2955279052257538, "kl": 2.337890625, "learning_rate": 1.3245365601953423e-05, "loss": 0.1166, "num_tokens": 723124477.0, "reward": 0.6143973469734192, "reward_std": 0.1845897138118744, "rewards/accuracy_reward/mean": 0.1294642868451774, "rewards/accuracy_reward/std": 0.29985080286860466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05815377924591303, "step": 1458 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 888.8214721679688, "completions/mean_terminated_length": 782.5771484375, "completions/min_length": 447.75, "completions/min_terminated_length": 447.75, "epoch": 0.4358150996938242, "grad_norm": 0.26831236481666565, "kl": 1.4228515625, "learning_rate": 1.3235045797636851e-05, "loss": 0.088, "num_tokens": 723594301.0, "reward": 0.7399553805589676, "reward_std": 0.15645504742860794, "rewards/accuracy_reward/mean": 0.24776785634458065, "rewards/accuracy_reward/std": 0.4172823131084442, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04052485013380647, "step": 1459 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 911.2500457763672, "completions/mean_terminated_length": 780.875244140625, "completions/min_length": 355.5, "completions/min_terminated_length": 355.5, "epoch": 0.4361138077813457, "grad_norm": 0.19858023524284363, "kl": 1.947265625, "learning_rate": 1.3224722143901503e-05, "loss": 0.0992, "num_tokens": 724072621.0, "reward": 0.6171875149011612, "reward_std": 0.16268732585012913, "rewards/accuracy_reward/mean": 0.1294642873108387, "rewards/accuracy_reward/std": 0.3299908861517906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.0542035810649395, "step": 1460 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 910.2745971679688, "completions/mean_terminated_length": 805.0943603515625, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "epoch": 0.43641251586886715, "grad_norm": 0.1350526660680771, "kl": 1.2587890625, "learning_rate": 1.3214394653031616e-05, "loss": 0.0741, "num_tokens": 724549496.0, "reward": 0.6082589626312256, "reward_std": 0.1711450032889843, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.31266900151968, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04182914597913623, "step": 1461 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 925.1027221679688, "completions/mean_terminated_length": 779.2986450195312, "completions/min_length": 367.25, "completions/min_terminated_length": 367.25, "epoch": 0.4367112239563886, "grad_norm": 0.2713891565799713, "kl": 1.203125, "learning_rate": 1.3204063337315997e-05, "loss": 0.056, "num_tokens": 725039686.0, "reward": 0.640066996216774, "reward_std": 0.13300147466361523, "rewards/accuracy_reward/mean": 0.1495535704307258, "rewards/accuracy_reward/std": 0.3249989524483681, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04517605667933822, "step": 1462 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 887.6786041259766, "completions/mean_terminated_length": 784.9418792724609, "completions/min_length": 401.25, "completions/min_terminated_length": 401.25, "epoch": 0.4370099320439101, "grad_norm": 0.22924058139324188, "kl": 0.8115234375, "learning_rate": 1.3193728209047993e-05, "loss": 0.0448, "num_tokens": 725508694.0, "reward": 0.6489955633878708, "reward_std": 0.12322984938509762, "rewards/accuracy_reward/mean": 0.1540178544819355, "rewards/accuracy_reward/std": 0.277776263654232, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 1463 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 911.4308471679688, "completions/mean_terminated_length": 796.6541595458984, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "epoch": 0.43730864013143156, "grad_norm": 0.3525222837924957, "kl": 1.80859375, "learning_rate": 1.3183389280525497e-05, "loss": 0.0938, "num_tokens": 725993735.0, "reward": 0.6383928805589676, "reward_std": 0.16802274622023106, "rewards/accuracy_reward/mean": 0.1495535741560161, "rewards/accuracy_reward/std": 0.3351389914751053, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051463617011904716, "step": 1464 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 883.0759429931641, "completions/mean_terminated_length": 778.0510559082031, "completions/min_length": 442.25, "completions/min_terminated_length": 442.25, "epoch": 0.43760734821895303, "grad_norm": 0.5461865663528442, "kl": 1.37890625, "learning_rate": 1.3173046564050923e-05, "loss": 0.0946, "num_tokens": 726459177.0, "reward": 0.6037946790456772, "reward_std": 0.15421191044151783, "rewards/accuracy_reward/mean": 0.11160714365541935, "rewards/accuracy_reward/std": 0.28787998110055923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043266257271170616, "step": 1465 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 893.7768249511719, "completions/mean_terminated_length": 777.4033203125, "completions/min_length": 346.25, "completions/min_terminated_length": 346.25, "epoch": 0.4379060563064745, "grad_norm": 0.17935076355934143, "kl": 1.1181640625, "learning_rate": 1.3162700071931185e-05, "loss": 0.0433, "num_tokens": 726931205.0, "reward": 0.575334832072258, "reward_std": 0.1150963231921196, "rewards/accuracy_reward/mean": 0.0825892835855484, "rewards/accuracy_reward/std": 0.22977416589856148, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040314854588359594, "step": 1466 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 877.0826263427734, "completions/mean_terminated_length": 764.1598205566406, "completions/min_length": 345.75, "completions/min_terminated_length": 345.75, "epoch": 0.43820476439399597, "grad_norm": 0.15767937898635864, "kl": 1.3583984375, "learning_rate": 1.31523498164777e-05, "loss": 0.0767, "num_tokens": 727409146.0, "reward": 0.5965401977300644, "reward_std": 0.1253706687130034, "rewards/accuracy_reward/mean": 0.1090029738843441, "rewards/accuracy_reward/std": 0.26451365649700165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1467 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 868.7388763427734, "completions/mean_terminated_length": 775.7639465332031, "completions/min_length": 359.25, "completions/min_terminated_length": 359.25, "epoch": 0.43850347248151744, "grad_norm": 0.4997192323207855, "kl": 1.095703125, "learning_rate": 1.3141995810006352e-05, "loss": 0.0591, "num_tokens": 727865429.0, "reward": 0.6601562798023224, "reward_std": 0.1473876480013132, "rewards/accuracy_reward/mean": 0.1681547611951828, "rewards/accuracy_reward/std": 0.3708391636610031, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.021947781555354595, "step": 1468 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 924.7500305175781, "completions/mean_terminated_length": 832.4723205566406, "completions/min_length": 469.25, "completions/min_terminated_length": 469.25, "epoch": 0.4388021805690389, "grad_norm": 0.24871036410331726, "kl": 1.830078125, "learning_rate": 1.3131638064837496e-05, "loss": 0.09, "num_tokens": 728353205.0, "reward": 0.5518973469734192, "reward_std": 0.12732254154980183, "rewards/accuracy_reward/mean": 0.06026785634458065, "rewards/accuracy_reward/std": 0.22250671312212944, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 1469 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 865.0134429931641, "completions/mean_terminated_length": 770.0711059570312, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "epoch": 0.4391008886565604, "grad_norm": 0.24401317536830902, "kl": 1.78125, "learning_rate": 1.3121276593295939e-05, "loss": 0.0936, "num_tokens": 728810795.0, "reward": 0.5580357313156128, "reward_std": 0.15438219159841537, "rewards/accuracy_reward/mean": 0.06696428498253226, "rewards/accuracy_reward/std": 0.2280024215579033, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 1470 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 901.638427734375, "completions/mean_terminated_length": 789.2754821777344, "completions/min_length": 374.25, "completions/min_terminated_length": 374.25, "epoch": 0.43939959674408186, "grad_norm": 0.19297514855861664, "kl": 1.685546875, "learning_rate": 1.3110911407710909e-05, "loss": 0.0836, "num_tokens": 729288681.0, "reward": 0.5502232313156128, "reward_std": 0.1071161013096571, "rewards/accuracy_reward/mean": 0.05803571455180645, "rewards/accuracy_reward/std": 0.2287282459437847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 1471 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 827.341552734375, "completions/mean_terminated_length": 726.2217102050781, "completions/min_length": 304.5, "completions/min_terminated_length": 304.5, "epoch": 0.4396983048316033, "grad_norm": 0.20211705565452576, "kl": 1.447265625, "learning_rate": 1.3100542520416068e-05, "loss": 0.0765, "num_tokens": 729732866.0, "reward": 0.667410746216774, "reward_std": 0.13642330840229988, "rewards/accuracy_reward/mean": 0.17410714272409678, "rewards/accuracy_reward/std": 0.3515707030892372, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.039343451615422964, "step": 1472 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 890.7835235595703, "completions/mean_terminated_length": 804.6275177001953, "completions/min_length": 381.75, "completions/min_terminated_length": 381.75, "epoch": 0.4399970129191248, "grad_norm": 0.24666230380535126, "kl": 1.287109375, "learning_rate": 1.3090169943749475e-05, "loss": 0.0701, "num_tokens": 730203009.0, "reward": 0.6350446790456772, "reward_std": 0.12412016652524471, "rewards/accuracy_reward/mean": 0.1406249995343387, "rewards/accuracy_reward/std": 0.32899220287799835, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03141601476818323, "step": 1473 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 909.6964721679688, "completions/mean_terminated_length": 801.2566833496094, "completions/min_length": 319.25, "completions/min_terminated_length": 319.25, "epoch": 0.44029572100664627, "grad_norm": 0.16121168434619904, "kl": 0.9951171875, "learning_rate": 1.3079793690053577e-05, "loss": 0.0453, "num_tokens": 730683353.0, "reward": 0.5809152126312256, "reward_std": 0.13239996694028378, "rewards/accuracy_reward/mean": 0.0848214291036129, "rewards/accuracy_reward/std": 0.26698702573776245, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1474 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 855.3817291259766, "completions/mean_terminated_length": 776.8669281005859, "completions/min_length": 472.25, "completions/min_terminated_length": 472.25, "epoch": 0.44059442909416774, "grad_norm": 0.27152684330940247, "kl": 1.1328125, "learning_rate": 1.3069413771675202e-05, "loss": 0.056, "num_tokens": 731140004.0, "reward": 0.5943080633878708, "reward_std": 0.1294957036152482, "rewards/accuracy_reward/mean": 0.10044643119908869, "rewards/accuracy_reward/std": 0.2617764212191105, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 1475 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 890.5446929931641, "completions/mean_terminated_length": 784.9620056152344, "completions/min_length": 423.5, "completions/min_terminated_length": 423.5, "epoch": 0.4408931371816892, "grad_norm": 0.20011143386363983, "kl": 0.74365234375, "learning_rate": 1.3059030200965536e-05, "loss": 0.0482, "num_tokens": 731617640.0, "reward": 0.7946428954601288, "reward_std": 0.22939623519778252, "rewards/accuracy_reward/mean": 0.2968750037252903, "rewards/accuracy_reward/std": 0.44686248153448105, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 1476 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 939.6495819091797, "completions/mean_terminated_length": 829.8221588134766, "completions/min_length": 461.5, "completions/min_terminated_length": 461.5, "epoch": 0.4411918452692107, "grad_norm": 0.37606632709503174, "kl": 0.97265625, "learning_rate": 1.3048642990280109e-05, "loss": 0.0516, "num_tokens": 732112795.0, "reward": 0.549107164144516, "reward_std": 0.10413273540325463, "rewards/accuracy_reward/mean": 0.055803571827709675, "rewards/accuracy_reward/std": 0.19294749200344086, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.038836000952869654, "step": 1477 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 883.1741485595703, "completions/mean_terminated_length": 750.6747741699219, "completions/min_length": 344.25, "completions/min_terminated_length": 344.25, "epoch": 0.44149055335673215, "grad_norm": 0.15863588452339172, "kl": 0.53369140625, "learning_rate": 1.3038252151978785e-05, "loss": 0.0299, "num_tokens": 732578377.0, "reward": 0.6473214775323868, "reward_std": 0.15773946605622768, "rewards/accuracy_reward/mean": 0.14955357182770967, "rewards/accuracy_reward/std": 0.33641041815280914, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.016042086761444807, "step": 1478 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 887.9397735595703, "completions/mean_terminated_length": 793.4041442871094, "completions/min_length": 413.5, "completions/min_terminated_length": 413.5, "epoch": 0.4417892614442536, "grad_norm": 0.1429094821214676, "kl": 0.750244140625, "learning_rate": 1.3027857698425748e-05, "loss": 0.0347, "num_tokens": 733046798.0, "reward": 0.5954241305589676, "reward_std": 0.10096491780132055, "rewards/accuracy_reward/mean": 0.09821428544819355, "rewards/accuracy_reward/std": 0.2883717939257622, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.01845060009509325, "step": 1479 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 914.0000457763672, "completions/mean_terminated_length": 801.5369110107422, "completions/min_length": 518.25, "completions/min_terminated_length": 518.25, "epoch": 0.4420879695317751, "grad_norm": 0.18962587416172028, "kl": 0.9287109375, "learning_rate": 1.3017459641989474e-05, "loss": 0.0512, "num_tokens": 733527998.0, "reward": 0.6456473469734192, "reward_std": 0.15097282454371452, "rewards/accuracy_reward/mean": 0.1517857159487903, "rewards/accuracy_reward/std": 0.3291698917746544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1480 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 890.654052734375, "completions/mean_terminated_length": 771.2007141113281, "completions/min_length": 394.5, "completions/min_terminated_length": 394.5, "epoch": 0.44238667761929656, "grad_norm": 0.25262001156806946, "kl": 0.9130859375, "learning_rate": 1.300705799504273e-05, "loss": 0.0469, "num_tokens": 733999475.0, "reward": 0.6071428805589676, "reward_std": 0.12551776599138975, "rewards/accuracy_reward/mean": 0.11383928591385484, "rewards/accuracy_reward/std": 0.24072560295462608, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03934345254674554, "step": 1481 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 877.6696929931641, "completions/mean_terminated_length": 756.5998687744141, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "epoch": 0.44268538570681804, "grad_norm": 0.1330820918083191, "kl": 0.662353515625, "learning_rate": 1.2996652769962567e-05, "loss": 0.0312, "num_tokens": 734465487.0, "reward": 0.5848214626312256, "reward_std": 0.09660470578819513, "rewards/accuracy_reward/mean": 0.08705357229337096, "rewards/accuracy_reward/std": 0.24910186603665352, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4977678582072258, "rewards/tag_count_reward/std": 0.020125597715377808, "step": 1482 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 921.3326263427734, "completions/mean_terminated_length": 764.412109375, "completions/min_length": 452.75, "completions/min_terminated_length": 452.75, "epoch": 0.4429840937943395, "grad_norm": 0.29503968358039856, "kl": 0.72607421875, "learning_rate": 1.2986243979130277e-05, "loss": 0.0311, "num_tokens": 734946244.0, "reward": 0.6657366454601288, "reward_std": 0.1467064470052719, "rewards/accuracy_reward/mean": 0.1718750037252903, "rewards/accuracy_reward/std": 0.3618788756430149, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1483 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 928.5312805175781, "completions/mean_terminated_length": 808.145751953125, "completions/min_length": 382.25, "completions/min_terminated_length": 382.25, "epoch": 0.443282801881861, "grad_norm": 0.1747237741947174, "kl": 0.8359375, "learning_rate": 1.2975831634931401e-05, "loss": 0.0389, "num_tokens": 735435362.0, "reward": 0.663504496216774, "reward_std": 0.17711912095546722, "rewards/accuracy_reward/mean": 0.1696428614668548, "rewards/accuracy_reward/std": 0.3372690677642822, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 1484 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 883.8861999511719, "completions/mean_terminated_length": 775.7474212646484, "completions/min_length": 379.5, "completions/min_terminated_length": 379.5, "epoch": 0.44358150996938245, "grad_norm": 0.2664146423339844, "kl": 1.2109375, "learning_rate": 1.296541574975571e-05, "loss": 0.0657, "num_tokens": 735905503.0, "reward": 0.5820312649011612, "reward_std": 0.1519574224948883, "rewards/accuracy_reward/mean": 0.09635416651144624, "rewards/accuracy_reward/std": 0.2801229692995548, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047717904672026634, "step": 1485 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 910.4754943847656, "completions/mean_terminated_length": 774.9134826660156, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "epoch": 0.44388021805690386, "grad_norm": 0.19381649792194366, "kl": 1.162109375, "learning_rate": 1.295499633599719e-05, "loss": 0.056, "num_tokens": 736388916.0, "reward": 0.7868303954601288, "reward_std": 0.21456721052527428, "rewards/accuracy_reward/mean": 0.29464285634458065, "rewards/accuracy_reward/std": 0.43689335137605667, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 1486 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 947.2857513427734, "completions/mean_terminated_length": 822.1172332763672, "completions/min_length": 445.5, "completions/min_terminated_length": 445.5, "epoch": 0.44417892614442533, "grad_norm": 0.1615096628665924, "kl": 1.60546875, "learning_rate": 1.2944573406054021e-05, "loss": 0.0755, "num_tokens": 736887540.0, "reward": 0.6981027126312256, "reward_std": 0.23070861026644707, "rewards/accuracy_reward/mean": 0.20758928172290325, "rewards/accuracy_reward/std": 0.3876953572034836, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 1487 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 911.5223693847656, "completions/mean_terminated_length": 807.8176422119141, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "epoch": 0.4444776342319468, "grad_norm": 0.19722627103328705, "kl": 1.220703125, "learning_rate": 1.2934146972328568e-05, "loss": 0.0678, "num_tokens": 737375374.0, "reward": 0.6149553805589676, "reward_std": 0.14653826504945755, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.3083269074559212, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 1488 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 930.591552734375, "completions/mean_terminated_length": 835.3427581787109, "completions/min_length": 519.0, "completions/min_terminated_length": 519.0, "epoch": 0.4447763423194683, "grad_norm": 0.2566300630569458, "kl": 1.5859375, "learning_rate": 1.2923717047227368e-05, "loss": 0.077, "num_tokens": 737861255.0, "reward": 0.6316964477300644, "reward_std": 0.1406896859407425, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.3406362123787403, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 1489 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 934.2723693847656, "completions/mean_terminated_length": 822.5308227539062, "completions/min_length": 538.0, "completions/min_terminated_length": 538.0, "epoch": 0.44507505040698975, "grad_norm": 0.3871816098690033, "kl": 2.20703125, "learning_rate": 1.2913283643161111e-05, "loss": 0.0949, "num_tokens": 738356097.0, "reward": 0.653459832072258, "reward_std": 0.2087763547897339, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3718881160020828, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05713389813899994, "step": 1490 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 900.0290679931641, "completions/mean_terminated_length": 779.6256408691406, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.4453737584945112, "grad_norm": 0.26747363805770874, "kl": 1.150390625, "learning_rate": 1.2902846772544625e-05, "loss": 0.0508, "num_tokens": 738825598.0, "reward": 0.7393973618745804, "reward_std": 0.13325253874063492, "rewards/accuracy_reward/mean": 0.24553571175783873, "rewards/accuracy_reward/std": 0.3855450302362442, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.036084157414734364, "step": 1491 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 931.0736999511719, "completions/mean_terminated_length": 815.0902099609375, "completions/min_length": 498.5, "completions/min_terminated_length": 498.5, "epoch": 0.4456724665820327, "grad_norm": 0.7809299826622009, "kl": 2.619140625, "learning_rate": 1.2892406447796866e-05, "loss": 0.1192, "num_tokens": 739313855.0, "reward": 0.6590402126312256, "reward_std": 0.19768104702234268, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.33347973972558975, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06301466468721628, "step": 1492 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 917.6518249511719, "completions/mean_terminated_length": 774.2903442382812, "completions/min_length": 487.75, "completions/min_terminated_length": 487.75, "epoch": 0.44597117466955416, "grad_norm": 0.3588181138038635, "kl": 1.943359375, "learning_rate": 1.2881962681340894e-05, "loss": 0.0873, "num_tokens": 739798547.0, "reward": 0.6489955484867096, "reward_std": 0.18193182349205017, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.34823935851454735, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04522894416004419, "step": 1493 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 933.1741333007812, "completions/mean_terminated_length": 827.7263336181641, "completions/min_length": 494.25, "completions/min_terminated_length": 494.25, "epoch": 0.44626988275707563, "grad_norm": 0.37601959705352783, "kl": 2.21484375, "learning_rate": 1.2871515485603877e-05, "loss": 0.1, "num_tokens": 740287713.0, "reward": 0.7332589626312256, "reward_std": 0.1910938285291195, "rewards/accuracy_reward/mean": 0.2477678619325161, "rewards/accuracy_reward/std": 0.41852303594350815, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05846718233078718, "step": 1494 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6450892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 962.0625457763672, "completions/mean_terminated_length": 846.8570861816406, "completions/min_length": 510.25, "completions/min_terminated_length": 510.25, "epoch": 0.4465685908445971, "grad_norm": 0.24326446652412415, "kl": 1.4873046875, "learning_rate": 1.2861064873017044e-05, "loss": 0.068, "num_tokens": 740791357.0, "reward": 0.6026785969734192, "reward_std": 0.1430395869538188, "rewards/accuracy_reward/mean": 0.11383928311988711, "rewards/accuracy_reward/std": 0.2814481370151043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04981599189341068, "step": 1495 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 919.8884429931641, "completions/mean_terminated_length": 802.8051452636719, "completions/min_length": 474.0, "completions/min_terminated_length": 474.0, "epoch": 0.4468672989321186, "grad_norm": 0.26053035259246826, "kl": 1.5546875, "learning_rate": 1.285061085601571e-05, "loss": 0.0715, "num_tokens": 741273739.0, "reward": 0.6283482387661934, "reward_std": 0.15322365704923868, "rewards/accuracy_reward/mean": 0.14062499371357262, "rewards/accuracy_reward/std": 0.2483330201357603, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05409308057278395, "step": 1496 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 926.6406707763672, "completions/mean_terminated_length": 814.5620727539062, "completions/min_length": 486.0, "completions/min_terminated_length": 486.0, "epoch": 0.44716600701964004, "grad_norm": 0.21961741149425507, "kl": 1.5234375, "learning_rate": 1.284015344703923e-05, "loss": 0.0698, "num_tokens": 741770858.0, "reward": 0.580357164144516, "reward_std": 0.11854254454374313, "rewards/accuracy_reward/mean": 0.09375000325962901, "rewards/accuracy_reward/std": 0.2639131061732769, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05548384413123131, "step": 1497 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 957.6808624267578, "completions/mean_terminated_length": 820.8125915527344, "completions/min_length": 424.25, "completions/min_terminated_length": 424.25, "epoch": 0.4474647151071615, "grad_norm": 0.30872273445129395, "kl": 1.4404296875, "learning_rate": 1.2829692658530993e-05, "loss": 0.0746, "num_tokens": 742279899.0, "reward": 0.5943080633878708, "reward_std": 0.178963053971529, "rewards/accuracy_reward/mean": 0.10714285913854837, "rewards/accuracy_reward/std": 0.2984249144792557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054577698931097984, "step": 1498 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 946.3750457763672, "completions/mean_terminated_length": 840.9912414550781, "completions/min_length": 506.25, "completions/min_terminated_length": 506.25, "epoch": 0.447763423194683, "grad_norm": 0.2705741226673126, "kl": 1.3837890625, "learning_rate": 1.2819228502938417e-05, "loss": 0.0669, "num_tokens": 742779491.0, "reward": 0.7114955484867096, "reward_std": 0.24301955848932266, "rewards/accuracy_reward/mean": 0.22098214738070965, "rewards/accuracy_reward/std": 0.39904655516147614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04757413361221552, "step": 1499 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 902.4553985595703, "completions/mean_terminated_length": 780.191162109375, "completions/min_length": 414.25, "completions/min_terminated_length": 414.25, "epoch": 0.44806213128220446, "grad_norm": 0.6567896604537964, "kl": 1.875, "learning_rate": 1.2808760992712923e-05, "loss": 0.1012, "num_tokens": 743255231.0, "reward": 0.664620578289032, "reward_std": 0.17860521003603935, "rewards/accuracy_reward/mean": 0.18303571455180645, "rewards/accuracy_reward/std": 0.38237909972667694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848171710968, "rewards/tag_count_reward/std": 0.06452203914523125, "step": 1500 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 912.5781707763672, "completions/mean_terminated_length": 808.3443298339844, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "epoch": 0.4483608393697259, "grad_norm": 0.5776202082633972, "kl": 1.794921875, "learning_rate": 1.2798290140309924e-05, "loss": 0.0808, "num_tokens": 743740434.0, "reward": 0.5574776977300644, "reward_std": 0.14558125846087933, "rewards/accuracy_reward/mean": 0.07589285681024194, "rewards/accuracy_reward/std": 0.25357047095894814, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06343489047139883, "step": 1501 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 939.9911193847656, "completions/mean_terminated_length": 833.5638275146484, "completions/min_length": 518.75, "completions/min_terminated_length": 518.75, "epoch": 0.4486595474572474, "grad_norm": 0.35916605591773987, "kl": 2.330078125, "learning_rate": 1.2787815958188806e-05, "loss": 0.1157, "num_tokens": 744241678.0, "reward": 0.5937500298023224, "reward_std": 0.15340529568493366, "rewards/accuracy_reward/mean": 0.11160714365541935, "rewards/accuracy_reward/std": 0.28890132904052734, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06532111950218678, "step": 1502 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 911.6004943847656, "completions/mean_terminated_length": 765.5900573730469, "completions/min_length": 436.5, "completions/min_terminated_length": 436.5, "epoch": 0.44895825554476887, "grad_norm": 0.24942976236343384, "kl": 2.03125, "learning_rate": 1.2777338458812924e-05, "loss": 0.0967, "num_tokens": 744727307.0, "reward": 0.5786830559372902, "reward_std": 0.13288756739348173, "rewards/accuracy_reward/mean": 0.09374999813735485, "rewards/accuracy_reward/std": 0.2490154281258583, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.055482905358076096, "step": 1503 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5446428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 915.1205749511719, "completions/mean_terminated_length": 790.8870849609375, "completions/min_length": 283.75, "completions/min_terminated_length": 283.75, "epoch": 0.44925696363229034, "grad_norm": 9.79325008392334, "kl": 2.86328125, "learning_rate": 1.2766857654649578e-05, "loss": 0.1336, "num_tokens": 745215985.0, "reward": 0.6132812798023224, "reward_std": 0.1719677336513996, "rewards/accuracy_reward/mean": 0.12723213993012905, "rewards/accuracy_reward/std": 0.29277707263827324, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.055549753829836845, "step": 1504 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 900.1272583007812, "completions/mean_terminated_length": 794.4303588867188, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "epoch": 0.4495556717198118, "grad_norm": 0.3733076751232147, "kl": 2.76171875, "learning_rate": 1.2756373558169992e-05, "loss": 0.1287, "num_tokens": 745686410.0, "reward": 0.5770089402794838, "reward_std": 0.14503622520714998, "rewards/accuracy_reward/mean": 0.10044642817229033, "rewards/accuracy_reward/std": 0.23586171120405197, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05805058777332306, "step": 1505 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 942.6473693847656, "completions/mean_terminated_length": 832.5146636962891, "completions/min_length": 559.0, "completions/min_terminated_length": 559.0, "epoch": 0.4498543798073333, "grad_norm": 0.6945921182632446, "kl": 2.96875, "learning_rate": 1.2745886181849325e-05, "loss": 0.1305, "num_tokens": 746178492.0, "reward": 0.6032366305589676, "reward_std": 0.11839994229376316, "rewards/accuracy_reward/mean": 0.11607142654247582, "rewards/accuracy_reward/std": 0.27994044311344624, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 1506 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 863.2009429931641, "completions/mean_terminated_length": 762.1736907958984, "completions/min_length": 391.25, "completions/min_terminated_length": 391.25, "epoch": 0.45015308789485475, "grad_norm": 0.38658738136291504, "kl": 2.24609375, "learning_rate": 1.2735395538166625e-05, "loss": 0.106, "num_tokens": 746639398.0, "reward": 0.5820312649011612, "reward_std": 0.11115999612957239, "rewards/accuracy_reward/mean": 0.09188987943343818, "rewards/accuracy_reward/std": 0.264058880507946, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983869820833, "step": 1507 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 911.0000457763672, "completions/mean_terminated_length": 817.2700042724609, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "epoch": 0.4504517959823762, "grad_norm": 0.38592198491096497, "kl": 2.646484375, "learning_rate": 1.2724901639604826e-05, "loss": 0.1351, "num_tokens": 747114934.0, "reward": 0.5691964477300644, "reward_std": 0.13045812025666237, "rewards/accuracy_reward/mean": 0.0825892873108387, "rewards/accuracy_reward/std": 0.2615179643034935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05581916682422161, "step": 1508 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 904.982177734375, "completions/mean_terminated_length": 804.9558258056641, "completions/min_length": 408.5, "completions/min_terminated_length": 408.5, "epoch": 0.4507505040698977, "grad_norm": 0.225722074508667, "kl": 2.328125, "learning_rate": 1.2714404498650743e-05, "loss": 0.1142, "num_tokens": 747590446.0, "reward": 0.6473214626312256, "reward_std": 0.134065181016922, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.35038840770721436, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 1509 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 900.4286193847656, "completions/mean_terminated_length": 790.4909057617188, "completions/min_length": 404.5, "completions/min_terminated_length": 404.5, "epoch": 0.45104921215741917, "grad_norm": 0.2601242661476135, "kl": 2.517578125, "learning_rate": 1.2703904127795052e-05, "loss": 0.1087, "num_tokens": 748069838.0, "reward": 0.5915178954601288, "reward_std": 0.18435887433588505, "rewards/accuracy_reward/mean": 0.10639881202951074, "rewards/accuracy_reward/std": 0.28231529518961906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.055022322572767735, "step": 1510 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 886.9330749511719, "completions/mean_terminated_length": 753.0791931152344, "completions/min_length": 361.25, "completions/min_terminated_length": 361.25, "epoch": 0.45134792024494064, "grad_norm": 0.30210092663764954, "kl": 2.517578125, "learning_rate": 1.2693400539532263e-05, "loss": 0.1531, "num_tokens": 748531104.0, "reward": 0.6590401828289032, "reward_std": 0.2229975163936615, "rewards/accuracy_reward/mean": 0.17410714365541935, "rewards/accuracy_reward/std": 0.3698214367032051, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05686598177999258, "step": 1511 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 892.5178985595703, "completions/mean_terminated_length": 768.9041595458984, "completions/min_length": 360.5, "completions/min_terminated_length": 360.5, "epoch": 0.4516466283324621, "grad_norm": 0.18940962851047516, "kl": 2.04296875, "learning_rate": 1.2682893746360716e-05, "loss": 0.1068, "num_tokens": 749014920.0, "reward": 0.6456473469734192, "reward_std": 0.1592635246925056, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3004690706729889, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04960631299763918, "step": 1512 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 827.4330749511719, "completions/mean_terminated_length": 729.9624176025391, "completions/min_length": 371.75, "completions/min_terminated_length": 371.75, "epoch": 0.4519453364199836, "grad_norm": 0.4558429718017578, "kl": 1.54296875, "learning_rate": 1.267238376078257e-05, "loss": 0.1092, "num_tokens": 749452810.0, "reward": 0.7656250298023224, "reward_std": 0.20024880021810532, "rewards/accuracy_reward/mean": 0.2745535708963871, "rewards/accuracy_reward/std": 0.43222104012966156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 1513 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 886.5669860839844, "completions/mean_terminated_length": 765.3135833740234, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "epoch": 0.45224404450750505, "grad_norm": 0.42652130126953125, "kl": 1.73828125, "learning_rate": 1.2661870595303777e-05, "loss": 0.0983, "num_tokens": 749925448.0, "reward": 0.5496652126312256, "reward_std": 0.12633121758699417, "rewards/accuracy_reward/mean": 0.06473214295692742, "rewards/accuracy_reward/std": 0.22686495445668697, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05014888383448124, "step": 1514 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 865.1473541259766, "completions/mean_terminated_length": 769.4850463867188, "completions/min_length": 430.0, "completions/min_terminated_length": 430.0, "epoch": 0.4525427525950265, "grad_norm": 0.191913440823555, "kl": 1.6005859375, "learning_rate": 1.2651354262434082e-05, "loss": 0.0822, "num_tokens": 750385226.0, "reward": 0.6166295111179352, "reward_std": 0.15760770067572594, "rewards/accuracy_reward/mean": 0.12500000116415322, "rewards/accuracy_reward/std": 0.2964470814913511, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.043475935235619545, "step": 1515 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 850.7344207763672, "completions/mean_terminated_length": 709.3287506103516, "completions/min_length": 302.75, "completions/min_terminated_length": 302.75, "epoch": 0.452841460682548, "grad_norm": 0.30745455622673035, "kl": 1.7177734375, "learning_rate": 1.2640834774686985e-05, "loss": 0.0798, "num_tokens": 750835475.0, "reward": 0.675223246216774, "reward_std": 0.1370295938104391, "rewards/accuracy_reward/mean": 0.1908482164144516, "rewards/accuracy_reward/std": 0.39061328023672104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 1516 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 888.2411041259766, "completions/mean_terminated_length": 785.8113708496094, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "epoch": 0.45314016877006946, "grad_norm": 0.18830397725105286, "kl": 1.263671875, "learning_rate": 1.2630312144579748e-05, "loss": 0.0762, "num_tokens": 751312559.0, "reward": 0.6551339626312256, "reward_std": 0.19180606678128242, "rewards/accuracy_reward/mean": 0.160714291036129, "rewards/accuracy_reward/std": 0.3390912339091301, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 1517 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 915.7344207763672, "completions/mean_terminated_length": 797.844482421875, "completions/min_length": 416.25, "completions/min_terminated_length": 416.25, "epoch": 0.45343887685759093, "grad_norm": 0.17439469695091248, "kl": 1.203125, "learning_rate": 1.2619786384633374e-05, "loss": 0.0672, "num_tokens": 751797608.0, "reward": 0.7472098469734192, "reward_std": 0.1828160658478737, "rewards/accuracy_reward/mean": 0.25223213993012905, "rewards/accuracy_reward/std": 0.4049060046672821, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.494977667927742, "rewards/tag_count_reward/std": 0.03507901635020971, "step": 1518 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 903.2991485595703, "completions/mean_terminated_length": 788.8683929443359, "completions/min_length": 348.5, "completions/min_terminated_length": 348.5, "epoch": 0.4537375849451124, "grad_norm": 0.29924532771110535, "kl": 1.3466796875, "learning_rate": 1.260925750737259e-05, "loss": 0.0774, "num_tokens": 752273246.0, "reward": 0.5675223469734192, "reward_std": 0.12053840793669224, "rewards/accuracy_reward/mean": 0.07366071455180645, "rewards/accuracy_reward/std": 0.219160545617342, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 1519 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 887.6406707763672, "completions/mean_terminated_length": 796.3194427490234, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "epoch": 0.4540362930326339, "grad_norm": 0.35449182987213135, "kl": 1.51171875, "learning_rate": 1.2598725525325823e-05, "loss": 0.08, "num_tokens": 752732221.0, "reward": 0.6930803954601288, "reward_std": 0.1371654663234949, "rewards/accuracy_reward/mean": 0.19866071455180645, "rewards/accuracy_reward/std": 0.3881432041525841, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.03659330774098635, "step": 1520 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 904.2478179931641, "completions/mean_terminated_length": 779.5829772949219, "completions/min_length": 362.75, "completions/min_terminated_length": 362.75, "epoch": 0.45433500112015535, "grad_norm": 0.45782470703125, "kl": 1.34765625, "learning_rate": 1.2588190451025209e-05, "loss": 0.0733, "num_tokens": 753212684.0, "reward": 0.6886160969734192, "reward_std": 0.1398604642599821, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.38475505262613297, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1521 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 908.9911193847656, "completions/mean_terminated_length": 804.4228057861328, "completions/min_length": 526.75, "completions/min_terminated_length": 526.75, "epoch": 0.4546337092076768, "grad_norm": 0.4117668569087982, "kl": 1.42431640625, "learning_rate": 1.257765229700655e-05, "loss": 0.0805, "num_tokens": 753697944.0, "reward": 0.5987723469734192, "reward_std": 0.13949810713529587, "rewards/accuracy_reward/mean": 0.10491071501746774, "rewards/accuracy_reward/std": 0.28661592677235603, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.031825252808630466, "step": 1522 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 915.7634429931641, "completions/mean_terminated_length": 807.7231750488281, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "epoch": 0.4549324172951983, "grad_norm": 0.17353732883930206, "kl": 1.365234375, "learning_rate": 1.2567111075809319e-05, "loss": 0.0715, "num_tokens": 754184462.0, "reward": 0.6462053805589676, "reward_std": 0.10587063990533352, "rewards/accuracy_reward/mean": 0.15178571757860482, "rewards/accuracy_reward/std": 0.3081010188907385, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 1523 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 909.3237152099609, "completions/mean_terminated_length": 777.615478515625, "completions/min_length": 322.5, "completions/min_terminated_length": 322.5, "epoch": 0.45523112538271976, "grad_norm": 0.17925213277339935, "kl": 1.595703125, "learning_rate": 1.2556566799976643e-05, "loss": 0.0772, "num_tokens": 754668591.0, "reward": 0.620535746216774, "reward_std": 0.18903028033673763, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.3189479038119316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 1524 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 901.7723693847656, "completions/mean_terminated_length": 774.3987579345703, "completions/min_length": 326.75, "completions/min_terminated_length": 326.75, "epoch": 0.45552983347024123, "grad_norm": 0.26856234669685364, "kl": 1.58984375, "learning_rate": 1.2546019482055276e-05, "loss": 0.0926, "num_tokens": 755141353.0, "reward": 0.7315848469734192, "reward_std": 0.1938191782683134, "rewards/accuracy_reward/mean": 0.23660713713616133, "rewards/accuracy_reward/std": 0.37704169005155563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 1525 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 911.8594207763672, "completions/mean_terminated_length": 791.1354217529297, "completions/min_length": 454.75, "completions/min_terminated_length": 454.75, "epoch": 0.4558285415577627, "grad_norm": 0.21890051662921906, "kl": 1.916015625, "learning_rate": 1.2535469134595598e-05, "loss": 0.1019, "num_tokens": 755620122.0, "reward": 0.6300223618745804, "reward_std": 0.16359291970729828, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.32066721469163895, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04909886047244072, "step": 1526 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 905.4866485595703, "completions/mean_terminated_length": 777.0142822265625, "completions/min_length": 379.75, "completions/min_terminated_length": 379.75, "epoch": 0.45612724964528417, "grad_norm": 0.24866946041584015, "kl": 1.4501953125, "learning_rate": 1.252491577015158e-05, "loss": 0.0658, "num_tokens": 756103140.0, "reward": 0.621651828289032, "reward_std": 0.1216726996935904, "rewards/accuracy_reward/mean": 0.1387648768723011, "rewards/accuracy_reward/std": 0.27838651090860367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1527 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 906.2344207763672, "completions/mean_terminated_length": 789.9091949462891, "completions/min_length": 443.25, "completions/min_terminated_length": 443.25, "epoch": 0.45642595773280564, "grad_norm": 0.20093047618865967, "kl": 1.111328125, "learning_rate": 1.2514359401280805e-05, "loss": 0.0537, "num_tokens": 756583869.0, "reward": 0.6891741454601288, "reward_std": 0.17891096323728561, "rewards/accuracy_reward/mean": 0.1964285708963871, "rewards/accuracy_reward/std": 0.39691826701164246, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 1528 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 922.6183471679688, "completions/mean_terminated_length": 800.0967102050781, "completions/min_length": 394.5, "completions/min_terminated_length": 394.5, "epoch": 0.45672466582032706, "grad_norm": 0.20717860758304596, "kl": 0.53125, "learning_rate": 1.2503800040544417e-05, "loss": 0.033, "num_tokens": 757070242.0, "reward": 0.6735491454601288, "reward_std": 0.151577889919281, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.37659741193056107, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 1529 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 902.5112152099609, "completions/mean_terminated_length": 785.5411224365234, "completions/min_length": 354.75, "completions/min_terminated_length": 354.75, "epoch": 0.45702337390784853, "grad_norm": 0.16989564895629883, "kl": 1.2529296875, "learning_rate": 1.2493237700507117e-05, "loss": 0.0619, "num_tokens": 757537751.0, "reward": 0.7332589626312256, "reward_std": 0.1897621862590313, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.4203283116221428, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047143861185759306, "step": 1530 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 911.2567291259766, "completions/mean_terminated_length": 802.8934326171875, "completions/min_length": 439.0, "completions/min_terminated_length": 439.0, "epoch": 0.45732208199537, "grad_norm": 0.22645603120326996, "kl": 0.99609375, "learning_rate": 1.2482672393737164e-05, "loss": 0.0508, "num_tokens": 758023130.0, "reward": 0.6763393133878708, "reward_std": 0.1618968192487955, "rewards/accuracy_reward/mean": 0.18303571455180645, "rewards/accuracy_reward/std": 0.3618747144937515, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1531 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 869.263427734375, "completions/mean_terminated_length": 720.2763366699219, "completions/min_length": 357.5, "completions/min_terminated_length": 357.5, "epoch": 0.45762079008289147, "grad_norm": 0.15789695084095, "kl": 0.93408203125, "learning_rate": 1.2472104132806338e-05, "loss": 0.0451, "num_tokens": 758485024.0, "reward": 0.6780134290456772, "reward_std": 0.11163358390331268, "rewards/accuracy_reward/mean": 0.18861607322469354, "rewards/accuracy_reward/std": 0.34066546335816383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 1532 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 942.7053833007812, "completions/mean_terminated_length": 839.0656127929688, "completions/min_length": 400.5, "completions/min_terminated_length": 400.5, "epoch": 0.45791949817041294, "grad_norm": 0.31345173716545105, "kl": 1.0341796875, "learning_rate": 1.2461532930289932e-05, "loss": 0.0484, "num_tokens": 758987868.0, "reward": 0.5440848395228386, "reward_std": 0.09536002576351166, "rewards/accuracy_reward/mean": 0.05468750116415322, "rewards/accuracy_reward/std": 0.1635278668254614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04791746288537979, "step": 1533 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 931.6361999511719, "completions/mean_terminated_length": 813.3393707275391, "completions/min_length": 410.75, "completions/min_terminated_length": 410.75, "epoch": 0.4582182062579344, "grad_norm": 0.31974655389785767, "kl": 1.0791015625, "learning_rate": 1.2450958798766748e-05, "loss": 0.0692, "num_tokens": 759479849.0, "reward": 0.6529018133878708, "reward_std": 0.23103927075862885, "rewards/accuracy_reward/mean": 0.16294642724096775, "rewards/accuracy_reward/std": 0.3531579300761223, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 1534 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 948.0067443847656, "completions/mean_terminated_length": 815.56884765625, "completions/min_length": 474.25, "completions/min_terminated_length": 474.25, "epoch": 0.4585169143454559, "grad_norm": 0.4205550253391266, "kl": 1.216796875, "learning_rate": 1.2440381750819066e-05, "loss": 0.0593, "num_tokens": 759974748.0, "reward": 0.5753348544239998, "reward_std": 0.12485338933765888, "rewards/accuracy_reward/mean": 0.08668154757469893, "rewards/accuracy_reward/std": 0.21644173935055733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 1535 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6116071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 920.5111999511719, "completions/mean_terminated_length": 762.8335571289062, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "epoch": 0.45881562243297735, "grad_norm": 0.4346384108066559, "kl": 1.0888671875, "learning_rate": 1.242980179903264e-05, "loss": 0.0487, "num_tokens": 760457761.0, "reward": 0.5887276977300644, "reward_std": 0.15698064491152763, "rewards/accuracy_reward/mean": 0.10788690741173923, "rewards/accuracy_reward/std": 0.2589004095643759, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04672335181385279, "step": 1536 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6316964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 938.4308319091797, "completions/mean_terminated_length": 793.5392150878906, "completions/min_length": 483.0, "completions/min_terminated_length": 483.0, "epoch": 0.4591143305204988, "grad_norm": 0.24442815780639648, "kl": 1.7578125, "learning_rate": 1.2419218955996677e-05, "loss": 0.0863, "num_tokens": 760945906.0, "reward": 0.5959821715950966, "reward_std": 0.12616475578397512, "rewards/accuracy_reward/mean": 0.11123512033373117, "rewards/accuracy_reward/std": 0.25245503708720207, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05623576045036316, "step": 1537 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6629464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 954.6808471679688, "completions/mean_terminated_length": 827.308349609375, "completions/min_length": 536.5, "completions/min_terminated_length": 536.5, "epoch": 0.4594130386080203, "grad_norm": 0.394077867269516, "kl": 1.91015625, "learning_rate": 1.2408633234303828e-05, "loss": 0.0903, "num_tokens": 761441283.0, "reward": 0.5290178805589676, "reward_std": 0.1298058908432722, "rewards/accuracy_reward/mean": 0.0446428582072258, "rewards/accuracy_reward/std": 0.1972733847796917, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.060698604211211205, "step": 1538 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 936.7098693847656, "completions/mean_terminated_length": 804.4968872070312, "completions/min_length": 482.25, "completions/min_terminated_length": 482.25, "epoch": 0.45971174669554177, "grad_norm": 0.3559039235115051, "kl": 2.6640625, "learning_rate": 1.2398044646550167e-05, "loss": 0.1252, "num_tokens": 761941185.0, "reward": 0.6104910969734192, "reward_std": 0.1546209929510951, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.2749655097723007, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.481026791036129, "rewards/tag_count_reward/std": 0.06640799902379513, "step": 1539 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 924.7678985595703, "completions/mean_terminated_length": 813.7406768798828, "completions/min_length": 438.5, "completions/min_terminated_length": 438.5, "epoch": 0.46001045478306324, "grad_norm": 0.22322089970111847, "kl": 2.470703125, "learning_rate": 1.2387453205335175e-05, "loss": 0.1231, "num_tokens": 762429657.0, "reward": 0.5943080484867096, "reward_std": 0.14889302849769592, "rewards/accuracy_reward/mean": 0.11160714225843549, "rewards/accuracy_reward/std": 0.2958207167685032, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06320485845208168, "step": 1540 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 908.4107666015625, "completions/mean_terminated_length": 779.1713562011719, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "epoch": 0.4603091628705847, "grad_norm": 0.2723781168460846, "kl": 2.982421875, "learning_rate": 1.2376858923261732e-05, "loss": 0.149, "num_tokens": 762908257.0, "reward": 0.5479910895228386, "reward_std": 0.1406312994658947, "rewards/accuracy_reward/mean": 0.06919643026776612, "rewards/accuracy_reward/std": 0.2375775333493948, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946343421936, "rewards/tag_count_reward/std": 0.06462238729000092, "step": 1541 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 932.6205749511719, "completions/mean_terminated_length": 832.1965026855469, "completions/min_length": 550.75, "completions/min_terminated_length": 550.75, "epoch": 0.4606078709581062, "grad_norm": 0.3072300851345062, "kl": 2.26953125, "learning_rate": 1.23662618129361e-05, "loss": 0.109, "num_tokens": 763402135.0, "reward": 0.624441996216774, "reward_std": 0.21053824573755264, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.34237825870513916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06167032290250063, "step": 1542 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6272321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 953.6094055175781, "completions/mean_terminated_length": 839.0118560791016, "completions/min_length": 412.75, "completions/min_terminated_length": 412.75, "epoch": 0.46090657904562765, "grad_norm": 0.2735268771648407, "kl": 2.76171875, "learning_rate": 1.2355661886967904e-05, "loss": 0.1233, "num_tokens": 763905000.0, "reward": 0.5647321790456772, "reward_std": 0.14130302146077156, "rewards/accuracy_reward/mean": 0.0803571455180645, "rewards/accuracy_reward/std": 0.25405827909708023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.060001716017723083, "step": 1543 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5558035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 931.4397888183594, "completions/mean_terminated_length": 824.4297027587891, "completions/min_length": 514.5, "completions/min_terminated_length": 514.5, "epoch": 0.4612052871331491, "grad_norm": 0.7771251797676086, "kl": 3.375, "learning_rate": 1.234505915797012e-05, "loss": 0.1572, "num_tokens": 764388605.0, "reward": 0.5909598618745804, "reward_std": 0.1561093684285879, "rewards/accuracy_reward/mean": 0.11160714458674192, "rewards/accuracy_reward/std": 0.30502165108919144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526828289032, "rewards/tag_count_reward/std": 0.06779913417994976, "step": 1544 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 885.6205596923828, "completions/mean_terminated_length": 771.8556976318359, "completions/min_length": 451.0, "completions/min_terminated_length": 451.0, "epoch": 0.4615039952206706, "grad_norm": 0.3605683743953705, "kl": 2.533203125, "learning_rate": 1.2334453638559057e-05, "loss": 0.1296, "num_tokens": 764858099.0, "reward": 0.671316996216774, "reward_std": 0.1760728396475315, "rewards/accuracy_reward/mean": 0.1852678619325161, "rewards/accuracy_reward/std": 0.3705590255558491, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05687962658703327, "step": 1545 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 925.2746124267578, "completions/mean_terminated_length": 817.0455322265625, "completions/min_length": 486.5, "completions/min_terminated_length": 486.5, "epoch": 0.46180270330819206, "grad_norm": 0.28301364183425903, "kl": 2.62109375, "learning_rate": 1.2323845341354347e-05, "loss": 0.1363, "num_tokens": 765349614.0, "reward": 0.5758928656578064, "reward_std": 0.14102441258728504, "rewards/accuracy_reward/mean": 0.09375000279396772, "rewards/accuracy_reward/std": 0.27151819318532944, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428507566452, "rewards/tag_count_reward/std": 0.06296313554048538, "step": 1546 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 905.7567443847656, "completions/mean_terminated_length": 786.9007415771484, "completions/min_length": 491.75, "completions/min_terminated_length": 491.75, "epoch": 0.46210141139571353, "grad_norm": 0.3883775770664215, "kl": 2.255859375, "learning_rate": 1.231323427897893e-05, "loss": 0.1148, "num_tokens": 765834305.0, "reward": 0.617745578289032, "reward_std": 0.14854652993381023, "rewards/accuracy_reward/mean": 0.1339285671710968, "rewards/accuracy_reward/std": 0.34065302461385727, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.061235176399350166, "step": 1547 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 923.4330749511719, "completions/mean_terminated_length": 808.3703002929688, "completions/min_length": 454.5, "completions/min_terminated_length": 454.5, "epoch": 0.462400119483235, "grad_norm": 0.2589973509311676, "kl": 1.9873046875, "learning_rate": 1.2302620464059026e-05, "loss": 0.096, "num_tokens": 766324611.0, "reward": 0.6054687798023224, "reward_std": 0.15107372216880322, "rewards/accuracy_reward/mean": 0.12053571501746774, "rewards/accuracy_reward/std": 0.291922889649868, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.0573029974475503, "step": 1548 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 907.4241485595703, "completions/mean_terminated_length": 747.5141296386719, "completions/min_length": 371.5, "completions/min_terminated_length": 371.5, "epoch": 0.4626988275707565, "grad_norm": 0.4240761697292328, "kl": 1.671875, "learning_rate": 1.2292003909224144e-05, "loss": 0.0775, "num_tokens": 766802737.0, "reward": 0.5446428805589676, "reward_std": 0.08252677135169506, "rewards/accuracy_reward/mean": 0.05803571594879031, "rewards/accuracy_reward/std": 0.18920787051320076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 1549 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 898.747802734375, "completions/mean_terminated_length": 809.1761627197266, "completions/min_length": 404.75, "completions/min_terminated_length": 404.75, "epoch": 0.46299753565827795, "grad_norm": 0.4478871822357178, "kl": 1.794921875, "learning_rate": 1.2281384627107045e-05, "loss": 0.0974, "num_tokens": 767275392.0, "reward": 0.6244419813156128, "reward_std": 0.18037651292979717, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3326601907610893, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057406721636652946, "step": 1550 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 897.0759429931641, "completions/mean_terminated_length": 794.5869293212891, "completions/min_length": 485.75, "completions/min_terminated_length": 485.75, "epoch": 0.4632962437457994, "grad_norm": 0.2442428171634674, "kl": 1.44140625, "learning_rate": 1.2270762630343734e-05, "loss": 0.0784, "num_tokens": 767748578.0, "reward": 0.6143973469734192, "reward_std": 0.1463559176772833, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.31858422607183456, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098951727152, "step": 1551 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 892.2902221679688, "completions/mean_terminated_length": 790.2123718261719, "completions/min_length": 381.75, "completions/min_terminated_length": 381.75, "epoch": 0.4635949518333209, "grad_norm": 0.22257967293262482, "kl": 1.88671875, "learning_rate": 1.2260137931573453e-05, "loss": 0.1029, "num_tokens": 768228148.0, "reward": 0.6746652126312256, "reward_std": 0.17096822895109653, "rewards/accuracy_reward/mean": 0.1875000004656613, "rewards/accuracy_reward/std": 0.32337306067347527, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487165167927742, "rewards/tag_count_reward/std": 0.05258398596197367, "step": 1552 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 885.4620971679688, "completions/mean_terminated_length": 776.1720581054688, "completions/min_length": 385.5, "completions/min_terminated_length": 385.5, "epoch": 0.46389365992084236, "grad_norm": 0.28431615233421326, "kl": 2.171875, "learning_rate": 1.2249510543438652e-05, "loss": 0.1106, "num_tokens": 768705043.0, "reward": 0.592075914144516, "reward_std": 0.19638903811573982, "rewards/accuracy_reward/mean": 0.1049107164144516, "rewards/accuracy_reward/std": 0.30459602922201157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054577698931097984, "step": 1553 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 930.2879943847656, "completions/mean_terminated_length": 806.0039215087891, "completions/min_length": 514.0, "completions/min_terminated_length": 514.0, "epoch": 0.46419236800836383, "grad_norm": 0.32915937900543213, "kl": 2.095703125, "learning_rate": 1.2238880478584987e-05, "loss": 0.1043, "num_tokens": 769215508.0, "reward": 0.6244419813156128, "reward_std": 0.1934336107224226, "rewards/accuracy_reward/mean": 0.13839285681024194, "rewards/accuracy_reward/std": 0.3141746260225773, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057152451016008854, "step": 1554 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 918.0781555175781, "completions/mean_terminated_length": 830.52001953125, "completions/min_length": 528.75, "completions/min_terminated_length": 528.75, "epoch": 0.4644910760958853, "grad_norm": 0.4031984806060791, "kl": 1.71484375, "learning_rate": 1.2228247749661293e-05, "loss": 0.0892, "num_tokens": 769700151.0, "reward": 0.6752232313156128, "reward_std": 0.19311894476413727, "rewards/accuracy_reward/mean": 0.18750000186264515, "rewards/accuracy_reward/std": 0.36854319646954536, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.053605979308485985, "step": 1555 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 914.3705749511719, "completions/mean_terminated_length": 804.3533630371094, "completions/min_length": 337.75, "completions/min_terminated_length": 337.75, "epoch": 0.4647897841834068, "grad_norm": 0.38620492815971375, "kl": 2.486328125, "learning_rate": 1.221761236931958e-05, "loss": 0.1218, "num_tokens": 770195069.0, "reward": 0.6607143133878708, "reward_std": 0.22215264290571213, "rewards/accuracy_reward/mean": 0.17410713993012905, "rewards/accuracy_reward/std": 0.3674593046307564, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05471411347389221, "step": 1556 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 912.3370819091797, "completions/mean_terminated_length": 793.9124755859375, "completions/min_length": 480.25, "completions/min_terminated_length": 480.25, "epoch": 0.46508849227092824, "grad_norm": 0.25332018733024597, "kl": 2.09375, "learning_rate": 1.2206974350215016e-05, "loss": 0.1027, "num_tokens": 770671748.0, "reward": 0.6696428954601288, "reward_std": 0.15129895694553852, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.3635092079639435, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 1557 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 939.8661193847656, "completions/mean_terminated_length": 828.12451171875, "completions/min_length": 468.5, "completions/min_terminated_length": 468.5, "epoch": 0.4653872003584497, "grad_norm": 0.2681836187839508, "kl": 1.572265625, "learning_rate": 1.2196333705005892e-05, "loss": 0.0754, "num_tokens": 771174824.0, "reward": 0.5697544813156128, "reward_std": 0.08277713414281607, "rewards/accuracy_reward/mean": 0.0818452388048172, "rewards/accuracy_reward/std": 0.23291177302598953, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 1558 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 916.3772735595703, "completions/mean_terminated_length": 782.4367065429688, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.4656859084459712, "grad_norm": 0.6212192177772522, "kl": 2.439453125, "learning_rate": 1.2185690446353646e-05, "loss": 0.1126, "num_tokens": 771659585.0, "reward": 0.587611623108387, "reward_std": 0.17246359214186668, "rewards/accuracy_reward/mean": 0.09821428474970162, "rewards/accuracy_reward/std": 0.2711102943867445, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04848270770162344, "step": 1559 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 938.0022888183594, "completions/mean_terminated_length": 825.5009918212891, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.46598461653349266, "grad_norm": 0.2465970367193222, "kl": 2.38671875, "learning_rate": 1.2175044586922815e-05, "loss": 0.1075, "num_tokens": 772157778.0, "reward": 0.5859375298023224, "reward_std": 0.13414448872208595, "rewards/accuracy_reward/mean": 0.09821428381837904, "rewards/accuracy_reward/std": 0.22282781265676022, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05065554520115256, "step": 1560 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 934.4687957763672, "completions/mean_terminated_length": 784.5983123779297, "completions/min_length": 422.75, "completions/min_terminated_length": 422.75, "epoch": 0.46628332462101413, "grad_norm": 0.24092118442058563, "kl": 2.060546875, "learning_rate": 1.2164396139381029e-05, "loss": 0.1051, "num_tokens": 772648692.0, "reward": 0.6088169887661934, "reward_std": 0.16159996949136257, "rewards/accuracy_reward/mean": 0.1183035746216774, "rewards/accuracy_reward/std": 0.25855783373117447, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.040441323071718216, "step": 1561 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 911.9643402099609, "completions/mean_terminated_length": 773.2062072753906, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.4665820327085356, "grad_norm": 0.19503501057624817, "kl": 2.42578125, "learning_rate": 1.2153745116399e-05, "loss": 0.1131, "num_tokens": 773124868.0, "reward": 0.5948660969734192, "reward_std": 0.1580425649881363, "rewards/accuracy_reward/mean": 0.10714285774156451, "rewards/accuracy_reward/std": 0.2580628953874111, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.0542035810649395, "step": 1562 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 866.5870971679688, "completions/mean_terminated_length": 749.4630432128906, "completions/min_length": 317.25, "completions/min_terminated_length": 317.25, "epoch": 0.46688074079605707, "grad_norm": 0.26436853408813477, "kl": 1.697265625, "learning_rate": 1.2143091530650508e-05, "loss": 0.0823, "num_tokens": 773578315.0, "reward": 0.6143973469734192, "reward_std": 0.12246101163327694, "rewards/accuracy_reward/mean": 0.12276785494759679, "rewards/accuracy_reward/std": 0.22104666754603386, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04508844017982483, "step": 1563 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 912.8817291259766, "completions/mean_terminated_length": 758.8337554931641, "completions/min_length": 411.25, "completions/min_terminated_length": 411.25, "epoch": 0.46717944888357854, "grad_norm": 0.3216649889945984, "kl": 1.9267578125, "learning_rate": 1.2132435394812377e-05, "loss": 0.0935, "num_tokens": 774056934.0, "reward": 0.6746651977300644, "reward_std": 0.11494997143745422, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.31662897020578384, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.038298643194139004, "step": 1564 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 913.5826263427734, "completions/mean_terminated_length": 789.357177734375, "completions/min_length": 397.0, "completions/min_terminated_length": 397.0, "epoch": 0.4674781569711, "grad_norm": 0.25300371646881104, "kl": 1.693359375, "learning_rate": 1.2121776721564465e-05, "loss": 0.0769, "num_tokens": 774539803.0, "reward": 0.658482164144516, "reward_std": 0.11634715739637613, "rewards/accuracy_reward/mean": 0.16555059212259948, "rewards/accuracy_reward/std": 0.3317698799073696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03410126734524965, "step": 1565 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.546875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 919.4844207763672, "completions/mean_terminated_length": 802.3939514160156, "completions/min_length": 446.5, "completions/min_terminated_length": 446.5, "epoch": 0.4677768650586215, "grad_norm": 0.18173423409461975, "kl": 1.255859375, "learning_rate": 1.2111115523589651e-05, "loss": 0.0534, "num_tokens": 775029156.0, "reward": 0.5630580484867096, "reward_std": 0.11107335146516562, "rewards/accuracy_reward/mean": 0.06919642933644354, "rewards/accuracy_reward/std": 0.230948057025671, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 1566 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 837.8103179931641, "completions/mean_terminated_length": 727.0459747314453, "completions/min_length": 363.75, "completions/min_terminated_length": 363.75, "epoch": 0.46807557314614295, "grad_norm": 0.23452714085578918, "kl": 1.439453125, "learning_rate": 1.2100451813573826e-05, "loss": 0.0831, "num_tokens": 775473983.0, "reward": 0.6785714477300644, "reward_std": 0.20543914660811424, "rewards/accuracy_reward/mean": 0.18749999813735485, "rewards/accuracy_reward/std": 0.3853197395801544, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589572083204985, "step": 1567 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 896.0580749511719, "completions/mean_terminated_length": 776.1987457275391, "completions/min_length": 373.5, "completions/min_terminated_length": 373.5, "epoch": 0.4683742812336644, "grad_norm": 0.24794472754001617, "kl": 1.1650390625, "learning_rate": 1.208978560420586e-05, "loss": 0.0583, "num_tokens": 775937321.0, "reward": 0.6506696790456772, "reward_std": 0.15927257761359215, "rewards/accuracy_reward/mean": 0.1584821450524032, "rewards/accuracy_reward/std": 0.33842485398054123, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04182914597913623, "step": 1568 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 881.3259429931641, "completions/mean_terminated_length": 751.0238342285156, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "epoch": 0.4686729893211859, "grad_norm": 0.18928250670433044, "kl": 1.0419921875, "learning_rate": 1.2079116908177592e-05, "loss": 0.0496, "num_tokens": 776398763.0, "reward": 0.632254496216774, "reward_std": 0.1941201239824295, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3415888100862503, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 1569 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 911.435302734375, "completions/mean_terminated_length": 796.3288116455078, "completions/min_length": 445.25, "completions/min_terminated_length": 445.25, "epoch": 0.46897169740870737, "grad_norm": 0.25083261728286743, "kl": 1.4296875, "learning_rate": 1.2068445738183843e-05, "loss": 0.0662, "num_tokens": 776877150.0, "reward": 0.628348246216774, "reward_std": 0.11048637516796589, "rewards/accuracy_reward/mean": 0.13616071292199194, "rewards/accuracy_reward/std": 0.2869419511407614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.03723818250000477, "step": 1570 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 913.3326263427734, "completions/mean_terminated_length": 793.9694976806641, "completions/min_length": 493.25, "completions/min_terminated_length": 493.25, "epoch": 0.46927040549622884, "grad_norm": 0.1928725689649582, "kl": 1.185546875, "learning_rate": 1.205777210692235e-05, "loss": 0.0561, "num_tokens": 777363795.0, "reward": 0.621651828289032, "reward_std": 0.13617053627967834, "rewards/accuracy_reward/mean": 0.12872023764066398, "rewards/accuracy_reward/std": 0.2629830129444599, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.036314870696514845, "step": 1571 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 934.0424499511719, "completions/mean_terminated_length": 826.9977416992188, "completions/min_length": 487.5, "completions/min_terminated_length": 487.5, "epoch": 0.46956911358375025, "grad_norm": 0.17522278428077698, "kl": 0.9375, "learning_rate": 1.2047096027093798e-05, "loss": 0.0406, "num_tokens": 777857430.0, "reward": 0.6724330633878708, "reward_std": 0.09508214797824621, "rewards/accuracy_reward/mean": 0.17857143026776612, "rewards/accuracy_reward/std": 0.3315699230879545, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.031692753080278635, "step": 1572 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 901.3125457763672, "completions/mean_terminated_length": 814.0452423095703, "completions/min_length": 421.75, "completions/min_terminated_length": 421.75, "epoch": 0.4698678216712717, "grad_norm": 0.33696115016937256, "kl": 1.216796875, "learning_rate": 1.2036417511401775e-05, "loss": 0.0745, "num_tokens": 778331650.0, "reward": 0.6177455633878708, "reward_std": 0.15270499885082245, "rewards/accuracy_reward/mean": 0.13132440578192472, "rewards/accuracy_reward/std": 0.3230040520429611, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04597289999946952, "step": 1573 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 876.3772583007812, "completions/mean_terminated_length": 769.22119140625, "completions/min_length": 403.75, "completions/min_terminated_length": 403.75, "epoch": 0.4701665297587932, "grad_norm": 0.3857272267341614, "kl": 0.675048828125, "learning_rate": 1.2025736572552775e-05, "loss": 0.04, "num_tokens": 778794139.0, "reward": 0.611607164144516, "reward_std": 0.08937890268862247, "rewards/accuracy_reward/mean": 0.11607142863795161, "rewards/accuracy_reward/std": 0.24801434576511383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.02858699206262827, "step": 1574 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 912.3370971679688, "completions/mean_terminated_length": 817.2119750976562, "completions/min_length": 504.25, "completions/min_terminated_length": 504.25, "epoch": 0.47046523784631467, "grad_norm": 0.3646295368671417, "kl": 0.8642578125, "learning_rate": 1.2015053223256173e-05, "loss": 0.0464, "num_tokens": 779277810.0, "reward": 0.5954241305589676, "reward_std": 0.136823495849967, "rewards/accuracy_reward/mean": 0.10491071408614516, "rewards/accuracy_reward/std": 0.2935599982738495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 1575 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39955357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 871.966552734375, "completions/mean_terminated_length": 776.4637145996094, "completions/min_length": 403.75, "completions/min_terminated_length": 403.75, "epoch": 0.47076394593383614, "grad_norm": 0.21301978826522827, "kl": 1.0693359375, "learning_rate": 1.2004367476224206e-05, "loss": 0.0473, "num_tokens": 779738179.0, "reward": 0.592075914144516, "reward_std": 0.158831387758255, "rewards/accuracy_reward/mean": 0.10044643003493547, "rewards/accuracy_reward/std": 0.2791781350970268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04293336346745491, "step": 1576 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 853.9420013427734, "completions/mean_terminated_length": 769.5126037597656, "completions/min_length": 393.5, "completions/min_terminated_length": 393.5, "epoch": 0.4710626540213576, "grad_norm": 0.18958529829978943, "kl": 1.0107421875, "learning_rate": 1.1993679344171973e-05, "loss": 0.051, "num_tokens": 780203033.0, "reward": 0.6501116454601288, "reward_std": 0.12839927151799202, "rewards/accuracy_reward/mean": 0.15625000395812094, "rewards/accuracy_reward/std": 0.3028471749275923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1577 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 860.5736999511719, "completions/mean_terminated_length": 772.0767669677734, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "epoch": 0.4713613621088791, "grad_norm": 0.2347380518913269, "kl": 1.763671875, "learning_rate": 1.1982988839817406e-05, "loss": 0.0858, "num_tokens": 780667258.0, "reward": 0.6383928656578064, "reward_std": 0.1539078876376152, "rewards/accuracy_reward/mean": 0.15513392724096775, "rewards/accuracy_reward/std": 0.3517383709549904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051717888563871384, "step": 1578 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 908.0357666015625, "completions/mean_terminated_length": 807.0827178955078, "completions/min_length": 473.5, "completions/min_terminated_length": 473.5, "epoch": 0.47166007019640055, "grad_norm": 0.24216751754283905, "kl": 1.5517578125, "learning_rate": 1.1972295975881263e-05, "loss": 0.0712, "num_tokens": 781149770.0, "reward": 0.5630580484867096, "reward_std": 0.08384046819992363, "rewards/accuracy_reward/mean": 0.07142857229337096, "rewards/accuracy_reward/std": 0.20832321792840958, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 1579 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 868.9085235595703, "completions/mean_terminated_length": 757.1320648193359, "completions/min_length": 449.75, "completions/min_terminated_length": 449.75, "epoch": 0.471958778283922, "grad_norm": 0.2375761717557907, "kl": 1.8935546875, "learning_rate": 1.19616007650871e-05, "loss": 0.0964, "num_tokens": 781607793.0, "reward": 0.5675223469734192, "reward_std": 0.11416259687393904, "rewards/accuracy_reward/mean": 0.08035714109428227, "rewards/accuracy_reward/std": 0.18669269233942032, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.04308177111670375, "step": 1580 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 868.7902221679688, "completions/mean_terminated_length": 754.1329345703125, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.4722574863714435, "grad_norm": 0.4126940071582794, "kl": 2.099609375, "learning_rate": 1.1950903220161286e-05, "loss": 0.1143, "num_tokens": 782064899.0, "reward": 0.667410746216774, "reward_std": 0.1835201419889927, "rewards/accuracy_reward/mean": 0.18080356903374195, "rewards/accuracy_reward/std": 0.35110941529273987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 1581 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 863.3906707763672, "completions/mean_terminated_length": 779.1468048095703, "completions/min_length": 425.75, "completions/min_terminated_length": 425.75, "epoch": 0.47255619445896496, "grad_norm": 0.4756726920604706, "kl": 1.97705078125, "learning_rate": 1.1940203353832943e-05, "loss": 0.0834, "num_tokens": 782525346.0, "reward": 0.5775669813156128, "reward_std": 0.11736485362052917, "rewards/accuracy_reward/mean": 0.08705357275903225, "rewards/accuracy_reward/std": 0.2402816116809845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04016849957406521, "step": 1582 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 861.4888763427734, "completions/mean_terminated_length": 778.9185180664062, "completions/min_length": 391.25, "completions/min_terminated_length": 391.25, "epoch": 0.47285490254648643, "grad_norm": 0.6650556921958923, "kl": 2.51953125, "learning_rate": 1.192950117883397e-05, "loss": 0.1234, "num_tokens": 782986413.0, "reward": 0.6551339477300644, "reward_std": 0.15549298375844955, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.34495872631669044, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053695037961006165, "step": 1583 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 904.3616485595703, "completions/mean_terminated_length": 817.2535247802734, "completions/min_length": 495.25, "completions/min_terminated_length": 495.25, "epoch": 0.4731536106340079, "grad_norm": 0.43830791115760803, "kl": 2.12109375, "learning_rate": 1.1918796707899016e-05, "loss": 0.1001, "num_tokens": 783463375.0, "reward": 0.6484375298023224, "reward_std": 0.235050231218338, "rewards/accuracy_reward/mean": 0.16071428917348385, "rewards/accuracy_reward/std": 0.3479287028312683, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05248269159346819, "step": 1584 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39508928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 879.9576263427734, "completions/mean_terminated_length": 784.7911834716797, "completions/min_length": 409.75, "completions/min_terminated_length": 409.75, "epoch": 0.4734523187215294, "grad_norm": 0.3992611765861511, "kl": 3.021484375, "learning_rate": 1.190808995376545e-05, "loss": 0.1542, "num_tokens": 783932092.0, "reward": 0.608816996216774, "reward_std": 0.14344551414251328, "rewards/accuracy_reward/mean": 0.1227678544819355, "rewards/accuracy_reward/std": 0.2676975652575493, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05585796386003494, "step": 1585 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 843.4397583007812, "completions/mean_terminated_length": 760.3204498291016, "completions/min_length": 367.25, "completions/min_terminated_length": 367.25, "epoch": 0.47375102680905085, "grad_norm": 0.1985652595758438, "kl": 2.048828125, "learning_rate": 1.1897380929173365e-05, "loss": 0.114, "num_tokens": 784378897.0, "reward": 0.7427455633878708, "reward_std": 0.18278245255351067, "rewards/accuracy_reward/mean": 0.25223213620483875, "rewards/accuracy_reward/std": 0.4081401601433754, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04517605667933822, "step": 1586 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 866.5736999511719, "completions/mean_terminated_length": 767.894287109375, "completions/min_length": 375.5, "completions/min_terminated_length": 375.5, "epoch": 0.4740497348965723, "grad_norm": 0.2792065739631653, "kl": 2.091796875, "learning_rate": 1.1886669646865554e-05, "loss": 0.126, "num_tokens": 784838594.0, "reward": 0.6545759290456772, "reward_std": 0.16969652473926544, "rewards/accuracy_reward/mean": 0.17447916232049465, "rewards/accuracy_reward/std": 0.3687223866581917, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04909886047244072, "step": 1587 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 854.529052734375, "completions/mean_terminated_length": 777.7480926513672, "completions/min_length": 349.5, "completions/min_terminated_length": 349.5, "epoch": 0.4743484429840938, "grad_norm": 0.260684072971344, "kl": 2.025390625, "learning_rate": 1.1875956119587499e-05, "loss": 0.1111, "num_tokens": 785294031.0, "reward": 0.6495535969734192, "reward_std": 0.12223058193922043, "rewards/accuracy_reward/mean": 0.16071428661234677, "rewards/accuracy_reward/std": 0.3066072575747967, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 1588 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 882.2946929931641, "completions/mean_terminated_length": 794.4775390625, "completions/min_length": 446.75, "completions/min_terminated_length": 446.75, "epoch": 0.47464715107161526, "grad_norm": 0.2200556844472885, "kl": 2.361328125, "learning_rate": 1.1865240360087349e-05, "loss": 0.1155, "num_tokens": 785765395.0, "reward": 0.5781250298023224, "reward_std": 0.08524662256240845, "rewards/accuracy_reward/mean": 0.09747023810632527, "rewards/accuracy_reward/std": 0.272454334422946, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.056180731393396854, "step": 1589 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 870.8683471679688, "completions/mean_terminated_length": 786.5898284912109, "completions/min_length": 390.25, "completions/min_terminated_length": 390.25, "epoch": 0.47494585915913673, "grad_norm": 0.30226120352745056, "kl": 2.1953125, "learning_rate": 1.185452238111591e-05, "loss": 0.112, "num_tokens": 786229000.0, "reward": 0.6344866305589676, "reward_std": 0.15937844291329384, "rewards/accuracy_reward/mean": 0.14732142654247582, "rewards/accuracy_reward/std": 0.31312222592532635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05430487543344498, "step": 1590 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 877.4397735595703, "completions/mean_terminated_length": 732.2131652832031, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "epoch": 0.4752445672466582, "grad_norm": 0.25922098755836487, "kl": 3.12109375, "learning_rate": 1.1843802195426634e-05, "loss": 0.16, "num_tokens": 786698557.0, "reward": 0.6411830484867096, "reward_std": 0.21533561870455742, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.36717963218688965, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48046875, "rewards/tag_count_reward/std": 0.06678942777216434, "step": 1591 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 888.9844055175781, "completions/mean_terminated_length": 767.1519927978516, "completions/min_length": 419.75, "completions/min_terminated_length": 419.75, "epoch": 0.47554327533417967, "grad_norm": 0.27317479252815247, "kl": 1.916015625, "learning_rate": 1.1833079815775596e-05, "loss": 0.0938, "num_tokens": 787173238.0, "reward": 0.6372768059372902, "reward_std": 0.15552034508436918, "rewards/accuracy_reward/mean": 0.14955356810241938, "rewards/accuracy_reward/std": 0.2634992450475693, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.0534067377448082, "step": 1592 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 876.5558319091797, "completions/mean_terminated_length": 777.3501892089844, "completions/min_length": 361.25, "completions/min_terminated_length": 361.25, "epoch": 0.47584198342170114, "grad_norm": 0.3255026042461395, "kl": 2.150390625, "learning_rate": 1.1822355254921478e-05, "loss": 0.1044, "num_tokens": 787634815.0, "reward": 0.719308078289032, "reward_std": 0.18351495265960693, "rewards/accuracy_reward/mean": 0.23437500232830644, "rewards/accuracy_reward/std": 0.3642292432487011, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05936532001942396, "step": 1593 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 921.1920166015625, "completions/mean_terminated_length": 801.7512512207031, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "epoch": 0.4761406915092226, "grad_norm": 0.5369585156440735, "kl": 1.94140625, "learning_rate": 1.1811628525625557e-05, "loss": 0.1036, "num_tokens": 788124389.0, "reward": 0.5122768208384514, "reward_std": 0.08380197919905186, "rewards/accuracy_reward/mean": 0.024553571362048388, "rewards/accuracy_reward/std": 0.10152360424399376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053695037961006165, "step": 1594 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 933.6786193847656, "completions/mean_terminated_length": 816.4869537353516, "completions/min_length": 491.0, "completions/min_terminated_length": 491.0, "epoch": 0.4764393995967441, "grad_norm": 0.46351534128189087, "kl": 2.80078125, "learning_rate": 1.1800899640651699e-05, "loss": 0.1352, "num_tokens": 788613861.0, "reward": 0.5965401977300644, "reward_std": 0.16738741844892502, "rewards/accuracy_reward/mean": 0.11383928591385484, "rewards/accuracy_reward/std": 0.2930401638150215, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06193623226135969, "step": 1595 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 931.9978179931641, "completions/mean_terminated_length": 815.5296020507812, "completions/min_length": 503.75, "completions/min_terminated_length": 503.75, "epoch": 0.47673810768426556, "grad_norm": 0.20390640199184418, "kl": 1.625, "learning_rate": 1.1790168612766331e-05, "loss": 0.0778, "num_tokens": 789098772.0, "reward": 0.6054687723517418, "reward_std": 0.173398912884295, "rewards/accuracy_reward/mean": 0.1160714291036129, "rewards/accuracy_reward/std": 0.26788436621427536, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04820432187989354, "step": 1596 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 899.2500457763672, "completions/mean_terminated_length": 788.0850067138672, "completions/min_length": 448.0, "completions/min_terminated_length": 448.0, "epoch": 0.477036815771787, "grad_norm": 0.3547343611717224, "kl": 2.11328125, "learning_rate": 1.177943545473842e-05, "loss": 0.1013, "num_tokens": 789570964.0, "reward": 0.6238839700818062, "reward_std": 0.12157446704804897, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.29218055307865143, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05782576743513346, "step": 1597 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 881.7567291259766, "completions/mean_terminated_length": 761.9942779541016, "completions/min_length": 404.75, "completions/min_terminated_length": 404.75, "epoch": 0.4773355238593085, "grad_norm": 0.329244464635849, "kl": 2.27734375, "learning_rate": 1.1768700179339484e-05, "loss": 0.1228, "num_tokens": 790042871.0, "reward": 0.662388414144516, "reward_std": 0.16129748336970806, "rewards/accuracy_reward/mean": 0.17410714831203222, "rewards/accuracy_reward/std": 0.3383273109793663, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.04972758423537016, "step": 1598 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49553571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 903.904052734375, "completions/mean_terminated_length": 787.92919921875, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "epoch": 0.47763423194682997, "grad_norm": 0.3702980875968933, "kl": 1.984375, "learning_rate": 1.1757962799343548e-05, "loss": 0.0962, "num_tokens": 790523196.0, "reward": 0.5524553805589676, "reward_std": 0.15420705266296864, "rewards/accuracy_reward/mean": 0.06250000023283064, "rewards/accuracy_reward/std": 0.2222723849117756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047143861185759306, "step": 1599 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 857.0022583007812, "completions/mean_terminated_length": 748.5214385986328, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "epoch": 0.47793294003435144, "grad_norm": 0.1983851045370102, "kl": 1.978515625, "learning_rate": 1.1747223327527149e-05, "loss": 0.0801, "num_tokens": 790979469.0, "reward": 0.5764509290456772, "reward_std": 0.12229789420962334, "rewards/accuracy_reward/mean": 0.09077380690723658, "rewards/accuracy_reward/std": 0.2363763302564621, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 1600 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 862.0893096923828, "completions/mean_terminated_length": 774.0263519287109, "completions/min_length": 419.5, "completions/min_terminated_length": 419.5, "epoch": 0.4782316481218729, "grad_norm": 0.5271719694137573, "kl": 1.8671875, "learning_rate": 1.1736481776669307e-05, "loss": 0.1098, "num_tokens": 791436773.0, "reward": 0.6099330633878708, "reward_std": 0.16952791810035706, "rewards/accuracy_reward/mean": 0.1205357126891613, "rewards/accuracy_reward/std": 0.32433729618787766, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04940675385296345, "step": 1601 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 871.0893096923828, "completions/mean_terminated_length": 771.138916015625, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "epoch": 0.4785303562093944, "grad_norm": 0.37069201469421387, "kl": 2.552734375, "learning_rate": 1.1725738159551518e-05, "loss": 0.1166, "num_tokens": 791906269.0, "reward": 0.5664062723517418, "reward_std": 0.10886758286505938, "rewards/accuracy_reward/mean": 0.07812500116415322, "rewards/accuracy_reward/std": 0.2038356326520443, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05223577655851841, "step": 1602 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 904.7589721679688, "completions/mean_terminated_length": 806.4236755371094, "completions/min_length": 449.25, "completions/min_terminated_length": 449.25, "epoch": 0.47882906429691585, "grad_norm": 0.41844794154167175, "kl": 2.611328125, "learning_rate": 1.1714992488957743e-05, "loss": 0.1189, "num_tokens": 792385633.0, "reward": 0.575334832072258, "reward_std": 0.11325666680932045, "rewards/accuracy_reward/mean": 0.0870535708963871, "rewards/accuracy_reward/std": 0.1885690689086914, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.044885930605232716, "step": 1603 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 889.3861846923828, "completions/mean_terminated_length": 773.6461944580078, "completions/min_length": 455.25, "completions/min_terminated_length": 455.25, "epoch": 0.4791277723844373, "grad_norm": 0.268093079328537, "kl": 2.1220703125, "learning_rate": 1.1704244777674377e-05, "loss": 0.1023, "num_tokens": 792853934.0, "reward": 0.6316964477300644, "reward_std": 0.15910056233406067, "rewards/accuracy_reward/mean": 0.14062500186264515, "rewards/accuracy_reward/std": 0.32547085359692574, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044403897132724524, "step": 1604 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 879.0312805175781, "completions/mean_terminated_length": 762.3809967041016, "completions/min_length": 364.5, "completions/min_terminated_length": 364.5, "epoch": 0.4794264804719588, "grad_norm": 0.1921159029006958, "kl": 2.326171875, "learning_rate": 1.1693495038490247e-05, "loss": 0.1092, "num_tokens": 793314108.0, "reward": 0.5641741305589676, "reward_std": 0.13961637392640114, "rewards/accuracy_reward/mean": 0.0770089291036129, "rewards/accuracy_reward/std": 0.2612738162279129, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052092005498707294, "step": 1605 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 893.9308319091797, "completions/mean_terminated_length": 781.3338165283203, "completions/min_length": 398.5, "completions/min_terminated_length": 398.5, "epoch": 0.47972518855948026, "grad_norm": 0.3099759817123413, "kl": 1.921875, "learning_rate": 1.1682743284196595e-05, "loss": 0.0924, "num_tokens": 793787613.0, "reward": 0.601004496216774, "reward_std": 0.1119012227281928, "rewards/accuracy_reward/mean": 0.10937499976716936, "rewards/accuracy_reward/std": 0.25946434773504734, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 1606 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 884.1428985595703, "completions/mean_terminated_length": 757.5220336914062, "completions/min_length": 368.5, "completions/min_terminated_length": 368.5, "epoch": 0.48002389664700174, "grad_norm": 0.24806472659111023, "kl": 2.0234375, "learning_rate": 1.1671989527587057e-05, "loss": 0.1031, "num_tokens": 794259661.0, "reward": 0.8175223618745804, "reward_std": 0.23809992522001266, "rewards/accuracy_reward/mean": 0.3303571455180645, "rewards/accuracy_reward/std": 0.4663940817117691, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 1607 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 901.4598541259766, "completions/mean_terminated_length": 768.7335510253906, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "epoch": 0.4803226047345232, "grad_norm": 0.17858891189098358, "kl": 1.7998046875, "learning_rate": 1.1661233781457655e-05, "loss": 0.0848, "num_tokens": 794734747.0, "reward": 0.5853794813156128, "reward_std": 0.12785048130899668, "rewards/accuracy_reward/mean": 0.09598214318975806, "rewards/accuracy_reward/std": 0.27150749042630196, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 1608 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 858.4486999511719, "completions/mean_terminated_length": 741.9349670410156, "completions/min_length": 308.25, "completions/min_terminated_length": 308.25, "epoch": 0.4806213128220447, "grad_norm": 0.38909703493118286, "kl": 1.798828125, "learning_rate": 1.1650476058606776e-05, "loss": 0.0901, "num_tokens": 795189108.0, "reward": 0.6255580633878708, "reward_std": 0.1606716699898243, "rewards/accuracy_reward/mean": 0.13169643096625805, "rewards/accuracy_reward/std": 0.3209913820028305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03752126870676875, "step": 1609 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 856.1518249511719, "completions/mean_terminated_length": 740.69775390625, "completions/min_length": 284.25, "completions/min_terminated_length": 284.25, "epoch": 0.48092002090956615, "grad_norm": 0.16915012896060944, "kl": 1.6806640625, "learning_rate": 1.1639716371835163e-05, "loss": 0.0853, "num_tokens": 795649656.0, "reward": 0.6261160969734192, "reward_std": 0.10276127606630325, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.33934778720140457, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 1610 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 871.4687805175781, "completions/mean_terminated_length": 786.0795593261719, "completions/min_length": 317.5, "completions/min_terminated_length": 317.5, "epoch": 0.4812187289970876, "grad_norm": 0.17987510561943054, "kl": 1.0458984375, "learning_rate": 1.162895473394589e-05, "loss": 0.0479, "num_tokens": 796112810.0, "reward": 0.7176339626312256, "reward_std": 0.14246198907494545, "rewards/accuracy_reward/mean": 0.22321428963914514, "rewards/accuracy_reward/std": 0.36466458812355995, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 1611 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 901.5960235595703, "completions/mean_terminated_length": 788.4648590087891, "completions/min_length": 288.0, "completions/min_terminated_length": 288.0, "epoch": 0.4815174370846091, "grad_norm": 0.22230352461338043, "kl": 1.724609375, "learning_rate": 1.1618191157744352e-05, "loss": 0.0819, "num_tokens": 796595589.0, "reward": 0.5976562947034836, "reward_std": 0.15591087006032467, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.30228496342897415, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 1612 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44419642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 901.2857666015625, "completions/mean_terminated_length": 803.2703399658203, "completions/min_length": 358.25, "completions/min_terminated_length": 358.25, "epoch": 0.48181614517213056, "grad_norm": 0.22465595602989197, "kl": 1.6005859375, "learning_rate": 1.1607425656038263e-05, "loss": 0.0841, "num_tokens": 797076405.0, "reward": 0.6450893133878708, "reward_std": 0.1829486135393381, "rewards/accuracy_reward/mean": 0.1562500037252903, "rewards/accuracy_reward/std": 0.34746944159269333, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.0499969981610775, "step": 1613 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 899.3437957763672, "completions/mean_terminated_length": 768.56982421875, "completions/min_length": 404.25, "completions/min_terminated_length": 404.25, "epoch": 0.48211485325965203, "grad_norm": 0.27587899565696716, "kl": 1.9765625, "learning_rate": 1.1596658241637612e-05, "loss": 0.0932, "num_tokens": 797551487.0, "reward": 0.6188616305589676, "reward_std": 0.14951784722507, "rewards/accuracy_reward/mean": 0.13169642817229033, "rewards/accuracy_reward/std": 0.3311108574271202, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05381597578525543, "step": 1614 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 867.622802734375, "completions/mean_terminated_length": 765.9275054931641, "completions/min_length": 341.75, "completions/min_terminated_length": 341.75, "epoch": 0.48241356134717345, "grad_norm": 0.35091543197631836, "kl": 2.486328125, "learning_rate": 1.1585888927354672e-05, "loss": 0.1313, "num_tokens": 798008598.0, "reward": 0.6551339626312256, "reward_std": 0.15076204389333725, "rewards/accuracy_reward/mean": 0.16964285634458065, "rewards/accuracy_reward/std": 0.37151313573122025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.057436331175267696, "step": 1615 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 904.3527221679688, "completions/mean_terminated_length": 799.681884765625, "completions/min_length": 411.5, "completions/min_terminated_length": 411.5, "epoch": 0.4827122694346949, "grad_norm": 0.3591032028198242, "kl": 1.4296875, "learning_rate": 1.1575117726003979e-05, "loss": 0.0758, "num_tokens": 798487492.0, "reward": 0.623325914144516, "reward_std": 0.14708933979272842, "rewards/accuracy_reward/mean": 0.13169643096625805, "rewards/accuracy_reward/std": 0.33425432443618774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1616 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 862.2545013427734, "completions/mean_terminated_length": 762.4305114746094, "completions/min_length": 280.75, "completions/min_terminated_length": 280.75, "epoch": 0.4830109775222164, "grad_norm": 0.3160060942173004, "kl": 2.71875, "learning_rate": 1.156434465040231e-05, "loss": 0.1478, "num_tokens": 798944790.0, "reward": 0.6869420111179352, "reward_std": 0.21116845309734344, "rewards/accuracy_reward/mean": 0.2031249962747097, "rewards/accuracy_reward/std": 0.39893771708011627, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06139749940484762, "step": 1617 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44419642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 883.6384429931641, "completions/mean_terminated_length": 768.4001312255859, "completions/min_length": 334.5, "completions/min_terminated_length": 334.5, "epoch": 0.48330968560973786, "grad_norm": 0.23889945447444916, "kl": 1.931640625, "learning_rate": 1.1553569713368672e-05, "loss": 0.0872, "num_tokens": 799413796.0, "reward": 0.7500000298023224, "reward_std": 0.1500732647255063, "rewards/accuracy_reward/mean": 0.2589285708963871, "rewards/accuracy_reward/std": 0.3481636792421341, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194977223873, "step": 1618 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 885.4777221679688, "completions/mean_terminated_length": 771.6307373046875, "completions/min_length": 336.5, "completions/min_terminated_length": 336.5, "epoch": 0.48360839369725933, "grad_norm": 0.3145331144332886, "kl": 2.193359375, "learning_rate": 1.154279292772429e-05, "loss": 0.1088, "num_tokens": 799878074.0, "reward": 0.6646205633878708, "reward_std": 0.19050734117627144, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.35069427639245987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04729470098391175, "step": 1619 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 854.0603179931641, "completions/mean_terminated_length": 757.0716705322266, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "epoch": 0.4839071017847808, "grad_norm": 0.33069878816604614, "kl": 2.068359375, "learning_rate": 1.1532014306292588e-05, "loss": 0.106, "num_tokens": 800338293.0, "reward": 0.6545759290456772, "reward_std": 0.21720874309539795, "rewards/accuracy_reward/mean": 0.16517857275903225, "rewards/accuracy_reward/std": 0.3605695888400078, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 1620 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 889.0937957763672, "completions/mean_terminated_length": 806.5642700195312, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "epoch": 0.4842058098723023, "grad_norm": 0.1628904789686203, "kl": 1.3427734375, "learning_rate": 1.1521233861899168e-05, "loss": 0.0708, "num_tokens": 800808111.0, "reward": 0.6467634290456772, "reward_std": 0.12673320062458515, "rewards/accuracy_reward/mean": 0.15401786006987095, "rewards/accuracy_reward/std": 0.3296068012714386, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.0401394609361887, "step": 1621 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 896.0826416015625, "completions/mean_terminated_length": 791.9419403076172, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.48450451795982374, "grad_norm": 0.2176012545824051, "kl": 2.169921875, "learning_rate": 1.1510451607371812e-05, "loss": 0.1073, "num_tokens": 801284788.0, "reward": 0.6964285969734192, "reward_std": 0.17210288532078266, "rewards/accuracy_reward/mean": 0.2075892835855484, "rewards/accuracy_reward/std": 0.4070097580552101, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 1622 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 812.3638763427734, "completions/mean_terminated_length": 728.0214996337891, "completions/min_length": 316.5, "completions/min_terminated_length": 316.5, "epoch": 0.4848032260473452, "grad_norm": 0.21751505136489868, "kl": 1.6484375, "learning_rate": 1.149966755554045e-05, "loss": 0.0822, "num_tokens": 801715911.0, "reward": 0.7366071790456772, "reward_std": 0.24636216089129448, "rewards/accuracy_reward/mean": 0.2570684552192688, "rewards/accuracy_reward/std": 0.43544451147317886, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04660273063927889, "step": 1623 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 846.2589569091797, "completions/mean_terminated_length": 757.2804107666016, "completions/min_length": 318.5, "completions/min_terminated_length": 318.5, "epoch": 0.4851019341348667, "grad_norm": 0.23682256042957306, "kl": 1.1923828125, "learning_rate": 1.1488881719237152e-05, "loss": 0.0654, "num_tokens": 802157019.0, "reward": 0.6824776977300644, "reward_std": 0.20054178312420845, "rewards/accuracy_reward/mean": 0.1875000037252903, "rewards/accuracy_reward/std": 0.38342566788196564, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 1624 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 890.6741333007812, "completions/mean_terminated_length": 784.0336456298828, "completions/min_length": 306.25, "completions/min_terminated_length": 306.25, "epoch": 0.48540064222238816, "grad_norm": 0.3178974986076355, "kl": 1.947265625, "learning_rate": 1.1478094111296109e-05, "loss": 0.0926, "num_tokens": 802630953.0, "reward": 0.5329241305589676, "reward_std": 0.09398236218839884, "rewards/accuracy_reward/mean": 0.04241071455180645, "rewards/accuracy_reward/std": 0.17141500860452652, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 1625 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 859.4018402099609, "completions/mean_terminated_length": 762.0144500732422, "completions/min_length": 378.25, "completions/min_terminated_length": 378.25, "epoch": 0.4856993503099096, "grad_norm": 0.24760808050632477, "kl": 1.6083984375, "learning_rate": 1.1467304744553618e-05, "loss": 0.1021, "num_tokens": 803081357.0, "reward": 0.7059151977300644, "reward_std": 0.1648847572505474, "rewards/accuracy_reward/mean": 0.21205357275903225, "rewards/accuracy_reward/std": 0.3840715065598488, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03667048690840602, "step": 1626 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 869.872802734375, "completions/mean_terminated_length": 727.6738739013672, "completions/min_length": 339.75, "completions/min_terminated_length": 339.75, "epoch": 0.4859980583974311, "grad_norm": 0.16374711692333221, "kl": 1.5810546875, "learning_rate": 1.1456513631848081e-05, "loss": 0.0976, "num_tokens": 803542580.0, "reward": 0.7299107313156128, "reward_std": 0.19068912416696548, "rewards/accuracy_reward/mean": 0.23660715110599995, "rewards/accuracy_reward/std": 0.41219815611839294, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.034409159794449806, "step": 1627 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41517857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 874.2611999511719, "completions/mean_terminated_length": 778.4532928466797, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "epoch": 0.48629676648495257, "grad_norm": 0.23230767250061035, "kl": 2.087890625, "learning_rate": 1.144572078601996e-05, "loss": 0.114, "num_tokens": 804000617.0, "reward": 0.6835937798023224, "reward_std": 0.15217985585331917, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.3927571550011635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 1628 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 864.9129943847656, "completions/mean_terminated_length": 765.6066589355469, "completions/min_length": 394.0, "completions/min_terminated_length": 394.0, "epoch": 0.48659547457247404, "grad_norm": 0.3407785892486572, "kl": 1.720703125, "learning_rate": 1.1434926219911792e-05, "loss": 0.0948, "num_tokens": 804461970.0, "reward": 0.6584821790456772, "reward_std": 0.23607610911130905, "rewards/accuracy_reward/mean": 0.18080357229337096, "rewards/accuracy_reward/std": 0.335625559091568, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 1629 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 869.4308319091797, "completions/mean_terminated_length": 762.6873168945312, "completions/min_length": 366.75, "completions/min_terminated_length": 366.75, "epoch": 0.4868941826599955, "grad_norm": 0.2671760618686676, "kl": 1.6416015625, "learning_rate": 1.1424129946368162e-05, "loss": 0.0944, "num_tokens": 804920755.0, "reward": 0.5937500298023224, "reward_std": 0.09841057006269693, "rewards/accuracy_reward/mean": 0.10044642770662904, "rewards/accuracy_reward/std": 0.21437203884124756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.036294152960181236, "step": 1630 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 863.8750457763672, "completions/mean_terminated_length": 741.3757781982422, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "epoch": 0.487192890747517, "grad_norm": 0.29175156354904175, "kl": 2.623046875, "learning_rate": 1.1413331978235677e-05, "loss": 0.1415, "num_tokens": 805382811.0, "reward": 0.6099330633878708, "reward_std": 0.15370009001344442, "rewards/accuracy_reward/mean": 0.12276785564608872, "rewards/accuracy_reward/std": 0.2993954475969076, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054504433646798134, "step": 1631 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6183035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 937.9754791259766, "completions/mean_terminated_length": 798.4433441162109, "completions/min_length": 381.25, "completions/min_terminated_length": 381.25, "epoch": 0.48749159883503845, "grad_norm": 0.1545017510652542, "kl": 1.4931640625, "learning_rate": 1.1402532328362965e-05, "loss": 0.0693, "num_tokens": 805875152.0, "reward": 0.5870535969734192, "reward_std": 0.08594589540734887, "rewards/accuracy_reward/mean": 0.09374999953433871, "rewards/accuracy_reward/std": 0.22816497832536697, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1632 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45535714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 895.2187957763672, "completions/mean_terminated_length": 787.5102081298828, "completions/min_length": 386.75, "completions/min_terminated_length": 386.75, "epoch": 0.4877903069225599, "grad_norm": 0.24428929388523102, "kl": 1.859375, "learning_rate": 1.1391731009600655e-05, "loss": 0.1029, "num_tokens": 806353218.0, "reward": 0.607700914144516, "reward_std": 0.1529453620314598, "rewards/accuracy_reward/mean": 0.11607142770662904, "rewards/accuracy_reward/std": 0.29686612263321877, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.0434872074984014, "step": 1633 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.0, "completions/mean_length": 874.0379943847656, "completions/mean_terminated_length": 778.9843139648438, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "epoch": 0.4880890150100814, "grad_norm": 0.1934749186038971, "kl": 1.71875, "learning_rate": 1.1380928034801366e-05, "loss": 0.0921, "num_tokens": 806820403.0, "reward": 0.676897332072258, "reward_std": 0.21635660529136658, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.38886042684316635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 1634 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 899.4933471679688, "completions/mean_terminated_length": 768.8834838867188, "completions/min_length": 366.75, "completions/min_terminated_length": 366.75, "epoch": 0.48838772309760287, "grad_norm": 0.25871044397354126, "kl": 2.04296875, "learning_rate": 1.1370123416819683e-05, "loss": 0.1055, "num_tokens": 807291648.0, "reward": 0.6847098618745804, "reward_std": 0.1725076138973236, "rewards/accuracy_reward/mean": 0.19419642724096775, "rewards/accuracy_reward/std": 0.38124360144138336, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04757413361221552, "step": 1635 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 891.7991638183594, "completions/mean_terminated_length": 799.4120483398438, "completions/min_length": 340.75, "completions/min_terminated_length": 340.75, "epoch": 0.48868643118512434, "grad_norm": 0.15740011632442474, "kl": 1.400390625, "learning_rate": 1.1359317168512143e-05, "loss": 0.0683, "num_tokens": 807768854.0, "reward": 0.650669664144516, "reward_std": 0.14922304265201092, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.3652483597397804, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 1636 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 882.2589721679688, "completions/mean_terminated_length": 788.7098846435547, "completions/min_length": 365.75, "completions/min_terminated_length": 365.75, "epoch": 0.4889851392726458, "grad_norm": 0.23645105957984924, "kl": 1.689453125, "learning_rate": 1.1348509302737232e-05, "loss": 0.0839, "num_tokens": 808238346.0, "reward": 0.6395089626312256, "reward_std": 0.13812627363950014, "rewards/accuracy_reward/mean": 0.1473214328289032, "rewards/accuracy_reward/std": 0.28809792548418045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 1637 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 891.0469207763672, "completions/mean_terminated_length": 778.9861145019531, "completions/min_length": 382.75, "completions/min_terminated_length": 382.75, "epoch": 0.4892838473601673, "grad_norm": 0.24015235900878906, "kl": 1.05859375, "learning_rate": 1.1337699832355354e-05, "loss": 0.0465, "num_tokens": 808709535.0, "reward": 0.601004496216774, "reward_std": 0.12629341520369053, "rewards/accuracy_reward/mean": 0.10491071781143546, "rewards/accuracy_reward/std": 0.26403576135635376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 1638 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 909.5558319091797, "completions/mean_terminated_length": 768.5944671630859, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "epoch": 0.48958255544768875, "grad_norm": 0.26366493105888367, "kl": 1.8134765625, "learning_rate": 1.1326888770228823e-05, "loss": 0.093, "num_tokens": 809189464.0, "reward": 0.6021205633878708, "reward_std": 0.16234212927520275, "rewards/accuracy_reward/mean": 0.11755952285602689, "rewards/accuracy_reward/std": 0.2999038062989712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04959508450701833, "step": 1639 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 907.1518249511719, "completions/mean_terminated_length": 789.0195159912109, "completions/min_length": 421.25, "completions/min_terminated_length": 421.25, "epoch": 0.4898812635352102, "grad_norm": 0.45332974195480347, "kl": 1.640625, "learning_rate": 1.1316076129221846e-05, "loss": 0.0791, "num_tokens": 809666220.0, "reward": 0.7187500298023224, "reward_std": 0.183514054864645, "rewards/accuracy_reward/mean": 0.2321428582072258, "rewards/accuracy_reward/std": 0.41888078302145004, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05070439958944917, "step": 1640 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 934.5870971679688, "completions/mean_terminated_length": 807.9862518310547, "completions/min_length": 467.0, "completions/min_terminated_length": 467.0, "epoch": 0.4901799716227317, "grad_norm": 0.17058172821998596, "kl": 1.328125, "learning_rate": 1.130526192220052e-05, "loss": 0.0636, "num_tokens": 810169875.0, "reward": 0.6741071939468384, "reward_std": 0.161215515807271, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.3815344497561455, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1641 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 875.0781707763672, "completions/mean_terminated_length": 754.4806518554688, "completions/min_length": 341.25, "completions/min_terminated_length": 341.25, "epoch": 0.49047867971025316, "grad_norm": 0.17513255774974823, "kl": 1.77490234375, "learning_rate": 1.1294446162032785e-05, "loss": 0.0931, "num_tokens": 810641222.0, "reward": 0.7003348469734192, "reward_std": 0.13727134559303522, "rewards/accuracy_reward/mean": 0.21391369332559407, "rewards/accuracy_reward/std": 0.34543844126164913, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04078465327620506, "step": 1642 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5982142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 926.0670013427734, "completions/mean_terminated_length": 782.5691680908203, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "epoch": 0.49077738779777463, "grad_norm": 0.1829720288515091, "kl": 1.5927734375, "learning_rate": 1.1283628861588444e-05, "loss": 0.0763, "num_tokens": 811128692.0, "reward": 0.623325914144516, "reward_std": 0.16779525950551033, "rewards/accuracy_reward/mean": 0.13169642724096775, "rewards/accuracy_reward/std": 0.31532423198223114, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.04348720656707883, "step": 1643 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 883.3058471679688, "completions/mean_terminated_length": 766.8977661132812, "completions/min_length": 377.75, "completions/min_terminated_length": 377.75, "epoch": 0.4910760958852961, "grad_norm": 0.32704243063926697, "kl": 1.76171875, "learning_rate": 1.1272810033739134e-05, "loss": 0.0947, "num_tokens": 811596509.0, "reward": 0.7293527126312256, "reward_std": 0.20674914866685867, "rewards/accuracy_reward/mean": 0.2388392835855484, "rewards/accuracy_reward/std": 0.40214546024799347, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 1644 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 882.8817291259766, "completions/mean_terminated_length": 767.1097106933594, "completions/min_length": 401.75, "completions/min_terminated_length": 401.75, "epoch": 0.4913748039728176, "grad_norm": 0.558361291885376, "kl": 1.4765625, "learning_rate": 1.12619896913583e-05, "loss": 0.0906, "num_tokens": 812076072.0, "reward": 0.6411830633878708, "reward_std": 0.12911652494221926, "rewards/accuracy_reward/mean": 0.1517857164144516, "rewards/accuracy_reward/std": 0.2966090813279152, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04940675385296345, "step": 1645 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 864.7790679931641, "completions/mean_terminated_length": 761.6640014648438, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.49167351206033905, "grad_norm": 0.26449766755104065, "kl": 1.5380859375, "learning_rate": 1.1251167847321194e-05, "loss": 0.0879, "num_tokens": 812546053.0, "reward": 0.5993303954601288, "reward_std": 0.14261114411056042, "rewards/accuracy_reward/mean": 0.1112351194024086, "rewards/accuracy_reward/std": 0.31550559401512146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.042172474320977926, "step": 1646 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 863.4040374755859, "completions/mean_terminated_length": 731.9253082275391, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 0.4919722201478605, "grad_norm": 0.1792672872543335, "kl": 1.611328125, "learning_rate": 1.1240344514504855e-05, "loss": 0.0856, "num_tokens": 813001722.0, "reward": 0.6668527126312256, "reward_std": 0.13482620380818844, "rewards/accuracy_reward/mean": 0.17410714738070965, "rewards/accuracy_reward/std": 0.3195885941386223, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 1647 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 870.9799499511719, "completions/mean_terminated_length": 766.447509765625, "completions/min_length": 365.75, "completions/min_terminated_length": 365.75, "epoch": 0.492270928235382, "grad_norm": 0.3310752511024475, "kl": 1.58984375, "learning_rate": 1.1229519705788094e-05, "loss": 0.0832, "num_tokens": 813465569.0, "reward": 0.668526828289032, "reward_std": 0.14246929250657558, "rewards/accuracy_reward/mean": 0.17857142724096775, "rewards/accuracy_reward/std": 0.36894287168979645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 1648 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 884.7053985595703, "completions/mean_terminated_length": 773.9409332275391, "completions/min_length": 405.75, "completions/min_terminated_length": 405.75, "epoch": 0.49256963632290346, "grad_norm": 0.3228301405906677, "kl": 1.720703125, "learning_rate": 1.1218693434051475e-05, "loss": 0.0774, "num_tokens": 813932221.0, "reward": 0.6679687798023224, "reward_std": 0.14706922881305218, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.35552164167165756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 1649 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 905.7344207763672, "completions/mean_terminated_length": 793.8341217041016, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "epoch": 0.49286834441042493, "grad_norm": 0.17066726088523865, "kl": 1.634765625, "learning_rate": 1.120786571217731e-05, "loss": 0.0877, "num_tokens": 814405158.0, "reward": 0.6300223469734192, "reward_std": 0.1400163769721985, "rewards/accuracy_reward/mean": 0.1383928549475968, "rewards/accuracy_reward/std": 0.31129785999655724, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.042347033973783255, "step": 1650 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 880.7701110839844, "completions/mean_terminated_length": 752.1598358154297, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "epoch": 0.4931670524979464, "grad_norm": 0.36274755001068115, "kl": 2.171875, "learning_rate": 1.1197036553049626e-05, "loss": 0.0942, "num_tokens": 814875295.0, "reward": 0.7137276977300644, "reward_std": 0.22808445990085602, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.41626792401075363, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.051475852727890015, "step": 1651 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 862.3125457763672, "completions/mean_terminated_length": 746.3406372070312, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "epoch": 0.49346576058546787, "grad_norm": 0.17766691744327545, "kl": 1.3818359375, "learning_rate": 1.118620596955417e-05, "loss": 0.0721, "num_tokens": 815334715.0, "reward": 0.6467634290456772, "reward_std": 0.1283509898930788, "rewards/accuracy_reward/mean": 0.16815476305782795, "rewards/accuracy_reward/std": 0.33747967705130577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03955313144251704, "step": 1652 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 908.2768249511719, "completions/mean_terminated_length": 796.3842620849609, "completions/min_length": 503.5, "completions/min_terminated_length": 503.5, "epoch": 0.49376446867298934, "grad_norm": 0.26932963728904724, "kl": 1.294921875, "learning_rate": 1.1175373974578378e-05, "loss": 0.0691, "num_tokens": 815812727.0, "reward": 0.6149553805589676, "reward_std": 0.1320108138024807, "rewards/accuracy_reward/mean": 0.1205357126891613, "rewards/accuracy_reward/std": 0.3099328354001045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1653 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 932.2679138183594, "completions/mean_terminated_length": 811.0116424560547, "completions/min_length": 409.25, "completions/min_terminated_length": 409.25, "epoch": 0.4940631767605108, "grad_norm": 0.3910362124443054, "kl": 1.951171875, "learning_rate": 1.1164540581011365e-05, "loss": 0.0935, "num_tokens": 816306511.0, "reward": 0.5948661118745804, "reward_std": 0.13463448733091354, "rewards/accuracy_reward/mean": 0.10714285750873387, "rewards/accuracy_reward/std": 0.2672754731029272, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 1654 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.5, "completions/mean_length": 807.5335388183594, "completions/mean_terminated_length": 696.5598602294922, "completions/min_length": 385.75, "completions/min_terminated_length": 385.75, "epoch": 0.4943618848480323, "grad_norm": 0.24613726139068604, "kl": 1.3388671875, "learning_rate": 1.115370580174392e-05, "loss": 0.0702, "num_tokens": 816743182.0, "reward": 0.6668527126312256, "reward_std": 0.10228491295129061, "rewards/accuracy_reward/mean": 0.17410714644938707, "rewards/accuracy_reward/std": 0.34555771946907043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818479284644, "step": 1655 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 904.3616485595703, "completions/mean_terminated_length": 783.7268829345703, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.49466059293555376, "grad_norm": 0.20705966651439667, "kl": 1.6630859375, "learning_rate": 1.1142869649668467e-05, "loss": 0.0796, "num_tokens": 817213744.0, "reward": 0.7187500298023224, "reward_std": 0.1536879539489746, "rewards/accuracy_reward/mean": 0.2299107126891613, "rewards/accuracy_reward/std": 0.41788625717163086, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.049996999092400074, "step": 1656 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 880.3795166015625, "completions/mean_terminated_length": 775.1310882568359, "completions/min_length": 461.5, "completions/min_terminated_length": 461.5, "epoch": 0.4949593010230752, "grad_norm": 0.4622337520122528, "kl": 1.23046875, "learning_rate": 1.113203213767907e-05, "loss": 0.0712, "num_tokens": 817681882.0, "reward": 0.6261160969734192, "reward_std": 0.1389194205403328, "rewards/accuracy_reward/mean": 0.13392857369035482, "rewards/accuracy_reward/std": 0.3184620812535286, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04132169345393777, "step": 1657 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 856.0558471679688, "completions/mean_terminated_length": 752.0047912597656, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.49525800911059664, "grad_norm": 0.6326101422309875, "kl": 1.90625, "learning_rate": 1.1121193278671409e-05, "loss": 0.1174, "num_tokens": 818140035.0, "reward": 0.6791294813156128, "reward_std": 0.16404365748167038, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.3935021907091141, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05430487543344498, "step": 1658 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 916.0245971679688, "completions/mean_terminated_length": 792.2347106933594, "completions/min_length": 340.75, "completions/min_terminated_length": 340.75, "epoch": 0.4955567171981181, "grad_norm": 0.24446946382522583, "kl": 2.53515625, "learning_rate": 1.1110353085542778e-05, "loss": 0.1247, "num_tokens": 818633662.0, "reward": 0.576450914144516, "reward_std": 0.1419951096177101, "rewards/accuracy_reward/mean": 0.09151785587891936, "rewards/accuracy_reward/std": 0.27847878634929657, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05874948389828205, "step": 1659 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 862.5111846923828, "completions/mean_terminated_length": 745.5021362304688, "completions/min_length": 394.5, "completions/min_terminated_length": 394.5, "epoch": 0.4958554252856396, "grad_norm": 0.37158361077308655, "kl": 3.1484375, "learning_rate": 1.1099511571192043e-05, "loss": 0.1593, "num_tokens": 819094883.0, "reward": 0.686941996216774, "reward_std": 0.1627490073442459, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.38873711228370667, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06442192755639553, "step": 1660 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 915.8013916015625, "completions/mean_terminated_length": 797.1707611083984, "completions/min_length": 422.25, "completions/min_terminated_length": 422.25, "epoch": 0.49615413337316105, "grad_norm": 0.27820587158203125, "kl": 3.3203125, "learning_rate": 1.1088668748519646e-05, "loss": 0.1569, "num_tokens": 819578602.0, "reward": 0.6383928805589676, "reward_std": 0.14521447010338306, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.351515457034111, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06410299427807331, "step": 1661 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 922.3839874267578, "completions/mean_terminated_length": 786.3740386962891, "completions/min_length": 372.5, "completions/min_terminated_length": 372.5, "epoch": 0.4964528414606825, "grad_norm": 0.7693492770195007, "kl": 4.703125, "learning_rate": 1.1077824630427593e-05, "loss": 0.2153, "num_tokens": 820062166.0, "reward": 0.5708705633878708, "reward_std": 0.13490816485136747, "rewards/accuracy_reward/mean": 0.09151785634458065, "rewards/accuracy_reward/std": 0.24624910205602646, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526753783226, "rewards/tag_count_reward/std": 0.06898579746484756, "step": 1662 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45758928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 870.5089569091797, "completions/mean_terminated_length": 750.7025146484375, "completions/min_length": 366.5, "completions/min_terminated_length": 366.5, "epoch": 0.496751549548204, "grad_norm": 0.473661333322525, "kl": 2.94921875, "learning_rate": 1.1066979229819427e-05, "loss": 0.1534, "num_tokens": 820523402.0, "reward": 0.6445312798023224, "reward_std": 0.149961456656456, "rewards/accuracy_reward/mean": 0.15624999860301614, "rewards/accuracy_reward/std": 0.3285013400018215, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05277834925800562, "step": 1663 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 911.9397735595703, "completions/mean_terminated_length": 784.5477447509766, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.49705025763572547, "grad_norm": 0.6060462594032288, "kl": 4.30859375, "learning_rate": 1.1056132559600216e-05, "loss": 0.2024, "num_tokens": 821002367.0, "reward": 0.6473214477300644, "reward_std": 0.16481850016862154, "rewards/accuracy_reward/mean": 0.16741071082651615, "rewards/accuracy_reward/std": 0.2935766950249672, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.479910708963871, "rewards/tag_count_reward/std": 0.06768756546080112, "step": 1664 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 886.4241485595703, "completions/mean_terminated_length": 763.8267059326172, "completions/min_length": 487.25, "completions/min_terminated_length": 487.25, "epoch": 0.49734896572324694, "grad_norm": 0.5617844462394714, "kl": 3.9765625, "learning_rate": 1.1045284632676535e-05, "loss": 0.1961, "num_tokens": 821480557.0, "reward": 0.6529018133878708, "reward_std": 0.21033627539873123, "rewards/accuracy_reward/mean": 0.17187499813735485, "rewards/accuracy_reward/std": 0.3678830936551094, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06413447111845016, "step": 1665 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 866.1562957763672, "completions/mean_terminated_length": 734.6078643798828, "completions/min_length": 377.5, "completions/min_terminated_length": 377.5, "epoch": 0.4976476738107684, "grad_norm": 0.5638870596885681, "kl": 3.21875, "learning_rate": 1.1034435461956465e-05, "loss": 0.173, "num_tokens": 821954627.0, "reward": 0.6121651977300644, "reward_std": 0.17127804458141327, "rewards/accuracy_reward/mean": 0.12499999743886292, "rewards/accuracy_reward/std": 0.29924397729337215, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055374542251229286, "step": 1666 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 896.2277221679688, "completions/mean_terminated_length": 787.9271087646484, "completions/min_length": 447.5, "completions/min_terminated_length": 447.5, "epoch": 0.4979463818982899, "grad_norm": 0.5026790499687195, "kl": 2.654296875, "learning_rate": 1.102358506034956e-05, "loss": 0.1214, "num_tokens": 822434889.0, "reward": 0.6400669813156128, "reward_std": 0.17004362493753433, "rewards/accuracy_reward/mean": 0.15401785727590322, "rewards/accuracy_reward/std": 0.3430631533265114, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05535051133483648, "step": 1667 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 877.6295013427734, "completions/mean_terminated_length": 775.8599853515625, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.49824508998581135, "grad_norm": 0.27923303842544556, "kl": 1.837890625, "learning_rate": 1.1012733440766834e-05, "loss": 0.12, "num_tokens": 822902083.0, "reward": 0.6484375298023224, "reward_std": 0.21728872135281563, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.3559955582022667, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04903263505548239, "step": 1668 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 888.8393249511719, "completions/mean_terminated_length": 757.1538238525391, "completions/min_length": 269.25, "completions/min_terminated_length": 269.25, "epoch": 0.4985437980733328, "grad_norm": 0.29622799158096313, "kl": 2.013671875, "learning_rate": 1.1001880616120764e-05, "loss": 0.1136, "num_tokens": 823367307.0, "reward": 0.6506696939468384, "reward_std": 0.1602102927863598, "rewards/accuracy_reward/mean": 0.16517857182770967, "rewards/accuracy_reward/std": 0.3448469266295433, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05674629285931587, "step": 1669 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 877.5803985595703, "completions/mean_terminated_length": 753.7438507080078, "completions/min_length": 426.25, "completions/min_terminated_length": 426.25, "epoch": 0.4988425061608543, "grad_norm": 0.17614975571632385, "kl": 1.806640625, "learning_rate": 1.0991026599325248e-05, "loss": 0.0996, "num_tokens": 823836959.0, "reward": 0.5602678656578064, "reward_std": 0.13792089093476534, "rewards/accuracy_reward/mean": 0.07142857345752418, "rewards/accuracy_reward/std": 0.22971452586352825, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05146361608058214, "step": 1670 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 877.1986999511719, "completions/mean_terminated_length": 745.6064147949219, "completions/min_length": 376.25, "completions/min_terminated_length": 376.25, "epoch": 0.49914121424837576, "grad_norm": 0.34207281470298767, "kl": 1.376953125, "learning_rate": 1.098017140329561e-05, "loss": 0.0865, "num_tokens": 824307272.0, "reward": 0.6294643133878708, "reward_std": 0.15875177085399628, "rewards/accuracy_reward/mean": 0.13839285913854837, "rewards/accuracy_reward/std": 0.3274182230234146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 1671 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 871.9710083007812, "completions/mean_terminated_length": 752.1867370605469, "completions/min_length": 278.75, "completions/min_terminated_length": 278.75, "epoch": 0.49943992233589724, "grad_norm": 0.21747270226478577, "kl": 1.646484375, "learning_rate": 1.0969315040948567e-05, "loss": 0.0807, "num_tokens": 824770107.0, "reward": 0.7304687947034836, "reward_std": 0.26185889542102814, "rewards/accuracy_reward/mean": 0.2433035671710968, "rewards/accuracy_reward/std": 0.42542557418346405, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05500977020710707, "step": 1672 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4821428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 897.6562957763672, "completions/mean_terminated_length": 779.6031646728516, "completions/min_length": 412.25, "completions/min_terminated_length": 412.25, "epoch": 0.4997386304234187, "grad_norm": 0.4032599627971649, "kl": 2.01171875, "learning_rate": 1.0958457525202241e-05, "loss": 0.1059, "num_tokens": 825248193.0, "reward": 0.6316964626312256, "reward_std": 0.15128101781010628, "rewards/accuracy_reward/mean": 0.14955357206054032, "rewards/accuracy_reward/std": 0.3226985912770033, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428656578064, "rewards/tag_count_reward/std": 0.06142285093665123, "step": 1673 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 855.1585083007812, "completions/mean_terminated_length": 752.6025543212891, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.5000373385109402, "grad_norm": 0.37713485956192017, "kl": 1.1640625, "learning_rate": 1.0947598868976113e-05, "loss": 0.0666, "num_tokens": 825704104.0, "reward": 0.7237723618745804, "reward_std": 0.19015947356820107, "rewards/accuracy_reward/mean": 0.24441964738070965, "rewards/accuracy_reward/std": 0.39927367866039276, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05165327340364456, "step": 1674 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 891.2522735595703, "completions/mean_terminated_length": 750.2187347412109, "completions/min_length": 309.25, "completions/min_terminated_length": 309.25, "epoch": 0.5003360465984616, "grad_norm": 0.3639819920063019, "kl": 1.0517578125, "learning_rate": 1.0936739085191025e-05, "loss": 0.0568, "num_tokens": 826175161.0, "reward": 0.621651828289032, "reward_std": 0.17539358139038086, "rewards/accuracy_reward/mean": 0.1294642877765, "rewards/accuracy_reward/std": 0.29794033989310265, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 1675 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 844.3058319091797, "completions/mean_terminated_length": 741.8050537109375, "completions/min_length": 341.25, "completions/min_terminated_length": 341.25, "epoch": 0.5006347546859832, "grad_norm": 0.6833979487419128, "kl": 1.7685546875, "learning_rate": 1.0925878186769159e-05, "loss": 0.1056, "num_tokens": 826626946.0, "reward": 0.6043527126312256, "reward_std": 0.14483586512506008, "rewards/accuracy_reward/mean": 0.11979166511446238, "rewards/accuracy_reward/std": 0.31347817555069923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.053723522927612066, "step": 1676 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 846.5960235595703, "completions/mean_terminated_length": 730.838134765625, "completions/min_length": 293.75, "completions/min_terminated_length": 293.75, "epoch": 0.5009334627735046, "grad_norm": 0.682964563369751, "kl": 1.826171875, "learning_rate": 1.0915016186634027e-05, "loss": 0.1111, "num_tokens": 827075981.0, "reward": 0.6635044813156128, "reward_std": 0.24067224189639091, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.34979820623993874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 1677 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 856.7143249511719, "completions/mean_terminated_length": 758.6848907470703, "completions/min_length": 390.75, "completions/min_terminated_length": 390.75, "epoch": 0.5012321708610261, "grad_norm": 0.6204670071601868, "kl": 2.140625, "learning_rate": 1.0904153097710446e-05, "loss": 0.1381, "num_tokens": 827529997.0, "reward": 0.656808078289032, "reward_std": 0.1542191468179226, "rewards/accuracy_reward/mean": 0.16964285634458065, "rewards/accuracy_reward/std": 0.36978935450315475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05382578168064356, "step": 1678 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.30580357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 810.5067291259766, "completions/mean_terminated_length": 721.1595916748047, "completions/min_length": 219.25, "completions/min_terminated_length": 219.25, "epoch": 0.5015308789485475, "grad_norm": 0.641867458820343, "kl": 1.60546875, "learning_rate": 1.0893288932924538e-05, "loss": 0.0721, "num_tokens": 827954784.0, "reward": 0.6077009290456772, "reward_std": 0.13583468738943338, "rewards/accuracy_reward/mean": 0.11160714225843549, "rewards/accuracy_reward/std": 0.2952858693897724, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.02676480822265148, "step": 1679 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 869.5960083007812, "completions/mean_terminated_length": 756.3263854980469, "completions/min_length": 326.25, "completions/min_terminated_length": 326.25, "epoch": 0.501829587036069, "grad_norm": 0.37986597418785095, "kl": 2.34375, "learning_rate": 1.0882423705203698e-05, "loss": 0.1226, "num_tokens": 828413163.0, "reward": 0.7343750149011612, "reward_std": 0.19542996399104595, "rewards/accuracy_reward/mean": 0.2455357201397419, "rewards/accuracy_reward/std": 0.42746806144714355, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050577715039253235, "step": 1680 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 846.2634429931641, "completions/mean_terminated_length": 745.2510223388672, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "epoch": 0.5021282951235905, "grad_norm": 0.2892000377178192, "kl": 2.58203125, "learning_rate": 1.0871557427476585e-05, "loss": 0.1334, "num_tokens": 828868433.0, "reward": 0.679129496216774, "reward_std": 0.18833313509821892, "rewards/accuracy_reward/mean": 0.1990327313542366, "rewards/accuracy_reward/std": 0.3960936740040779, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.049098861403763294, "step": 1681 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 835.138427734375, "completions/mean_terminated_length": 745.5781555175781, "completions/min_length": 342.75, "completions/min_terminated_length": 342.75, "epoch": 0.5024270032111119, "grad_norm": 0.5618308782577515, "kl": 2.22265625, "learning_rate": 1.0860690112673109e-05, "loss": 0.113, "num_tokens": 829317919.0, "reward": 0.652901828289032, "reward_std": 0.18109264597296715, "rewards/accuracy_reward/mean": 0.16071428265422583, "rewards/accuracy_reward/std": 0.3278949409723282, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 1682 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 850.9219055175781, "completions/mean_terminated_length": 766.0129699707031, "completions/min_length": 372.0, "completions/min_terminated_length": 372.0, "epoch": 0.5027257112986334, "grad_norm": 0.7888495922088623, "kl": 3.486328125, "learning_rate": 1.0849821773724419e-05, "loss": 0.1647, "num_tokens": 829767644.0, "reward": 0.5970982313156128, "reward_std": 0.1462068296968937, "rewards/accuracy_reward/mean": 0.11160714039579034, "rewards/accuracy_reward/std": 0.2944731153547764, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05895798094570637, "step": 1683 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 833.0893249511719, "completions/mean_terminated_length": 759.7739410400391, "completions/min_length": 358.75, "completions/min_terminated_length": 358.75, "epoch": 0.5030244193861548, "grad_norm": 0.438295841217041, "kl": 2.1171875, "learning_rate": 1.0838952423562877e-05, "loss": 0.1131, "num_tokens": 830215204.0, "reward": 0.6177455633878708, "reward_std": 0.15571920946240425, "rewards/accuracy_reward/mean": 0.12500000279396772, "rewards/accuracy_reward/std": 0.31049805879592896, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04015073226764798, "step": 1684 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.265625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 819.6875457763672, "completions/mean_terminated_length": 745.1701507568359, "completions/min_length": 362.75, "completions/min_terminated_length": 362.75, "epoch": 0.5033231274736764, "grad_norm": 0.2188778966665268, "kl": 2.076171875, "learning_rate": 1.0828082075122044e-05, "loss": 0.12, "num_tokens": 830661064.0, "reward": 0.6411830633878708, "reward_std": 0.15410234965384007, "rewards/accuracy_reward/mean": 0.1495535708963871, "rewards/accuracy_reward/std": 0.3427072502672672, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 1685 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 825.2611846923828, "completions/mean_terminated_length": 738.5301361083984, "completions/min_length": 376.75, "completions/min_terminated_length": 376.75, "epoch": 0.5036218355611978, "grad_norm": 0.1994311511516571, "kl": 1.8359375, "learning_rate": 1.0817210741336684e-05, "loss": 0.093, "num_tokens": 831109549.0, "reward": 0.5814732313156128, "reward_std": 0.16572687029838562, "rewards/accuracy_reward/mean": 0.08928571594879031, "rewards/accuracy_reward/std": 0.25451139733195305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 1686 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 868.6205749511719, "completions/mean_terminated_length": 788.6285705566406, "completions/min_length": 438.25, "completions/min_terminated_length": 438.25, "epoch": 0.5039205436487193, "grad_norm": 0.19940824806690216, "kl": 2.30078125, "learning_rate": 1.0806338435142718e-05, "loss": 0.1182, "num_tokens": 831573827.0, "reward": 0.6088169813156128, "reward_std": 0.19397810846567154, "rewards/accuracy_reward/mean": 0.12276785727590322, "rewards/accuracy_reward/std": 0.3194040209054947, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05612159986048937, "step": 1687 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 859.9152221679688, "completions/mean_terminated_length": 760.7976226806641, "completions/min_length": 442.0, "completions/min_terminated_length": 442.0, "epoch": 0.5042192517362407, "grad_norm": 0.3139062821865082, "kl": 1.294921875, "learning_rate": 1.0795465169477233e-05, "loss": 0.0827, "num_tokens": 832030685.0, "reward": 0.6590402126312256, "reward_std": 0.16428561136126518, "rewards/accuracy_reward/mean": 0.16741070989519358, "rewards/accuracy_reward/std": 0.35095780342817307, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04508844017982483, "step": 1688 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 858.0089569091797, "completions/mean_terminated_length": 774.2174682617188, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "epoch": 0.5045179598237622, "grad_norm": 0.4465380012989044, "kl": 1.705078125, "learning_rate": 1.0784590957278452e-05, "loss": 0.1042, "num_tokens": 832487185.0, "reward": 0.6261160969734192, "reward_std": 0.13257383415475488, "rewards/accuracy_reward/mean": 0.13616071827709675, "rewards/accuracy_reward/std": 0.27789144217967987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.048545535653829575, "step": 1689 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 882.7969055175781, "completions/mean_terminated_length": 777.6865844726562, "completions/min_length": 368.5, "completions/min_terminated_length": 368.5, "epoch": 0.5048166679112837, "grad_norm": 0.4360443651676178, "kl": 1.439453125, "learning_rate": 1.0773715811485728e-05, "loss": 0.0808, "num_tokens": 832955462.0, "reward": 0.585937537252903, "reward_std": 0.13169853202998638, "rewards/accuracy_reward/mean": 0.09374999906867743, "rewards/accuracy_reward/std": 0.22487326711416245, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.04197291610762477, "step": 1690 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 894.9754791259766, "completions/mean_terminated_length": 775.3529357910156, "completions/min_length": 298.75, "completions/min_terminated_length": 298.75, "epoch": 0.5051153759988052, "grad_norm": 0.2046210616827011, "kl": 1.40234375, "learning_rate": 1.0762839745039526e-05, "loss": 0.0714, "num_tokens": 833427835.0, "reward": 0.6238839626312256, "reward_std": 0.13253399170935154, "rewards/accuracy_reward/mean": 0.13169642654247582, "rewards/accuracy_reward/std": 0.29500251449644566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 1691 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5803571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 936.8705902099609, "completions/mean_terminated_length": 819.9257049560547, "completions/min_length": 399.5, "completions/min_terminated_length": 399.5, "epoch": 0.5054140840863266, "grad_norm": 0.23589931428432465, "kl": 1.2978515625, "learning_rate": 1.0751962770881401e-05, "loss": 0.0709, "num_tokens": 833921121.0, "reward": 0.572544664144516, "reward_std": 0.10067416587844491, "rewards/accuracy_reward/mean": 0.08035714272409678, "rewards/accuracy_reward/std": 0.22188352048397064, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04272336792200804, "step": 1692 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 932.3638763427734, "completions/mean_terminated_length": 835.0461883544922, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "epoch": 0.5057127921738481, "grad_norm": 0.32710304856300354, "kl": 1.5390625, "learning_rate": 1.0741084901953995e-05, "loss": 0.0762, "num_tokens": 834411860.0, "reward": 0.6060268133878708, "reward_std": 0.12384548713453114, "rewards/accuracy_reward/mean": 0.11383928917348385, "rewards/accuracy_reward/std": 0.25285572558641434, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 1693 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4821428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 890.466552734375, "completions/mean_terminated_length": 775.7502136230469, "completions/min_length": 345.5, "completions/min_terminated_length": 345.5, "epoch": 0.5060115002613695, "grad_norm": 0.43281280994415283, "kl": 1.419921875, "learning_rate": 1.0730206151201008e-05, "loss": 0.0867, "num_tokens": 834878453.0, "reward": 0.5814732313156128, "reward_std": 0.1081117745488882, "rewards/accuracy_reward/mean": 0.0892857126891613, "rewards/accuracy_reward/std": 0.28100351244211197, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043574150651693344, "step": 1694 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 910.2143096923828, "completions/mean_terminated_length": 790.8314666748047, "completions/min_length": 423.25, "completions/min_terminated_length": 423.25, "epoch": 0.5063102083488911, "grad_norm": 0.2812965512275696, "kl": 1.935546875, "learning_rate": 1.0719326531567195e-05, "loss": 0.1037, "num_tokens": 835360789.0, "reward": 0.6852678805589676, "reward_std": 0.19037288427352905, "rewards/accuracy_reward/mean": 0.1964285708963871, "rewards/accuracy_reward/std": 0.3895137384533882, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392761349678, "rewards/tag_count_reward/std": 0.04965366888791323, "step": 1695 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 899.2611999511719, "completions/mean_terminated_length": 767.1504821777344, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "epoch": 0.5066089164364125, "grad_norm": 0.23767398297786713, "kl": 1.6640625, "learning_rate": 1.0708446055998342e-05, "loss": 0.0847, "num_tokens": 835834250.0, "reward": 0.5747768133878708, "reward_std": 0.11905221687629819, "rewards/accuracy_reward/mean": 0.08258928591385484, "rewards/accuracy_reward/std": 0.21103477850556374, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 1696 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 882.5335235595703, "completions/mean_terminated_length": 774.5907592773438, "completions/min_length": 361.75, "completions/min_terminated_length": 361.75, "epoch": 0.506907624523934, "grad_norm": 0.21608683466911316, "kl": 1.1328125, "learning_rate": 1.0697564737441254e-05, "loss": 0.0604, "num_tokens": 836299689.0, "reward": 0.7042410969734192, "reward_std": 0.12872540391981602, "rewards/accuracy_reward/mean": 0.2098214328289032, "rewards/accuracy_reward/std": 0.40513667464256287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 1697 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 902.0469360351562, "completions/mean_terminated_length": 774.0886383056641, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "epoch": 0.5072063326114554, "grad_norm": 0.26712504029273987, "kl": 1.6484375, "learning_rate": 1.0686682588843737e-05, "loss": 0.0847, "num_tokens": 836787566.0, "reward": 0.6177455633878708, "reward_std": 0.1506650596857071, "rewards/accuracy_reward/mean": 0.12500000139698386, "rewards/accuracy_reward/std": 0.29783785343170166, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 1698 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 904.7277221679688, "completions/mean_terminated_length": 777.3563079833984, "completions/min_length": 387.75, "completions/min_terminated_length": 387.75, "epoch": 0.507505040698977, "grad_norm": 0.35252800583839417, "kl": 2.5390625, "learning_rate": 1.0675799623154593e-05, "loss": 0.1343, "num_tokens": 837267460.0, "reward": 0.619419664144516, "reward_std": 0.1826435923576355, "rewards/accuracy_reward/mean": 0.13430059514939785, "rewards/accuracy_reward/std": 0.3368005082011223, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 1699 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 889.8839569091797, "completions/mean_terminated_length": 772.0908660888672, "completions/min_length": 435.75, "completions/min_terminated_length": 435.75, "epoch": 0.5078037487864984, "grad_norm": 0.2662900388240814, "kl": 2.294921875, "learning_rate": 1.0664915853323581e-05, "loss": 0.1164, "num_tokens": 837738864.0, "reward": 0.5725446715950966, "reward_std": 0.1503293663263321, "rewards/accuracy_reward/mean": 0.08482142956927419, "rewards/accuracy_reward/std": 0.20667075738310814, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 1700 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 901.4442443847656, "completions/mean_terminated_length": 794.735595703125, "completions/min_length": 349.25, "completions/min_terminated_length": 349.25, "epoch": 0.5081024568740199, "grad_norm": 0.26520732045173645, "kl": 3.015625, "learning_rate": 1.0654031292301432e-05, "loss": 0.1491, "num_tokens": 838214967.0, "reward": 0.6512276977300644, "reward_std": 0.20390233397483826, "rewards/accuracy_reward/mean": 0.1674107126891613, "rewards/accuracy_reward/std": 0.3600342199206352, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.05926159583032131, "step": 1701 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.24553571428571427, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 816.7366485595703, "completions/mean_terminated_length": 751.4242553710938, "completions/min_length": 381.25, "completions/min_terminated_length": 381.25, "epoch": 0.5084011649615413, "grad_norm": 0.24670901894569397, "kl": 1.673828125, "learning_rate": 1.0643145953039811e-05, "loss": 0.0896, "num_tokens": 838655953.0, "reward": 0.6690848618745804, "reward_std": 0.1599325193092227, "rewards/accuracy_reward/mean": 0.17633928474970162, "rewards/accuracy_reward/std": 0.3361524026840925, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.042059858329594135, "step": 1702 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 878.9576110839844, "completions/mean_terminated_length": 787.5407104492188, "completions/min_length": 407.5, "completions/min_terminated_length": 407.5, "epoch": 0.5086998730490628, "grad_norm": 0.25146985054016113, "kl": 2.50390625, "learning_rate": 1.0632259848491307e-05, "loss": 0.1254, "num_tokens": 839122414.0, "reward": 0.6930803805589676, "reward_std": 0.1534719355404377, "rewards/accuracy_reward/mean": 0.20535714831203222, "rewards/accuracy_reward/std": 0.3751545175909996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 1703 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 851.2187957763672, "completions/mean_terminated_length": 781.2797698974609, "completions/min_length": 377.25, "completions/min_terminated_length": 377.25, "epoch": 0.5089985811365842, "grad_norm": 0.24930496513843536, "kl": 1.52294921875, "learning_rate": 1.062137299160943e-05, "loss": 0.0832, "num_tokens": 839570000.0, "reward": 0.6456473618745804, "reward_std": 0.13285164535045624, "rewards/accuracy_reward/mean": 0.15178571734577417, "rewards/accuracy_reward/std": 0.3344362899661064, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.030388458166271448, "step": 1704 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 867.9241485595703, "completions/mean_terminated_length": 763.6107177734375, "completions/min_length": 273.5, "completions/min_terminated_length": 273.5, "epoch": 0.5092972892241058, "grad_norm": 0.4645950198173523, "kl": 2.1064453125, "learning_rate": 1.0610485395348571e-05, "loss": 0.1065, "num_tokens": 840026510.0, "reward": 0.6651785969734192, "reward_std": 0.13480847142636776, "rewards/accuracy_reward/mean": 0.18229166604578495, "rewards/accuracy_reward/std": 0.3745551109313965, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04355311533436179, "step": 1705 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.21428571428571427, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 793.9643249511719, "completions/mean_terminated_length": 734.6334838867188, "completions/min_length": 318.25, "completions/min_terminated_length": 318.25, "epoch": 0.5095959973116272, "grad_norm": 0.3732717037200928, "kl": 1.4462890625, "learning_rate": 1.0599597072664012e-05, "loss": 0.075, "num_tokens": 840453870.0, "reward": 0.6741071790456772, "reward_std": 0.1336624976247549, "rewards/accuracy_reward/mean": 0.1785714311990887, "rewards/accuracy_reward/std": 0.29537750594317913, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.032084173522889614, "step": 1706 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 822.5692291259766, "completions/mean_terminated_length": 749.9631958007812, "completions/min_length": 309.5, "completions/min_terminated_length": 309.5, "epoch": 0.5098947053991487, "grad_norm": 0.3534054160118103, "kl": 2.3828125, "learning_rate": 1.058870803651189e-05, "loss": 0.1365, "num_tokens": 840886285.0, "reward": 0.5876116305589676, "reward_std": 0.17853389494121075, "rewards/accuracy_reward/mean": 0.0982142873108387, "rewards/accuracy_reward/std": 0.27508749067783356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.047245155554264784, "step": 1707 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 838.9665679931641, "completions/mean_terminated_length": 734.6641540527344, "completions/min_length": 255.75, "completions/min_terminated_length": 255.75, "epoch": 0.5101934134866701, "grad_norm": 0.3120653033256531, "kl": 2.54296875, "learning_rate": 1.0577818299849206e-05, "loss": 0.1297, "num_tokens": 841333326.0, "reward": 0.6886160969734192, "reward_std": 0.15882442891597748, "rewards/accuracy_reward/mean": 0.20312499813735485, "rewards/accuracy_reward/std": 0.37390774860978127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05846718233078718, "step": 1708 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 818.1719207763672, "completions/mean_terminated_length": 737.8239898681641, "completions/min_length": 316.25, "completions/min_terminated_length": 316.25, "epoch": 0.5104921215741917, "grad_norm": 0.43107765913009644, "kl": 1.931640625, "learning_rate": 1.0566927875633776e-05, "loss": 0.0922, "num_tokens": 841770619.0, "reward": 0.6456473395228386, "reward_std": 0.15665909415110946, "rewards/accuracy_reward/mean": 0.1540178544819355, "rewards/accuracy_reward/std": 0.3004781976342201, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 1709 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 845.2254791259766, "completions/mean_terminated_length": 770.4267578125, "completions/min_length": 324.75, "completions/min_terminated_length": 324.75, "epoch": 0.5107908296617131, "grad_norm": 0.24193544685840607, "kl": 1.837890625, "learning_rate": 1.0556036776824245e-05, "loss": 0.1025, "num_tokens": 842213232.0, "reward": 0.6819196715950966, "reward_std": 0.14540023356676102, "rewards/accuracy_reward/mean": 0.19196428544819355, "rewards/accuracy_reward/std": 0.3080570325255394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 1710 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.21651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 801.8036193847656, "completions/mean_terminated_length": 741.2826232910156, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "epoch": 0.5110895377492346, "grad_norm": 0.37150710821151733, "kl": 1.1904296875, "learning_rate": 1.0545145016380065e-05, "loss": 0.0669, "num_tokens": 842647032.0, "reward": 0.7287946939468384, "reward_std": 0.20115599408745766, "rewards/accuracy_reward/mean": 0.2343750037252903, "rewards/accuracy_reward/std": 0.4218292310833931, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1711 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 853.4821929931641, "completions/mean_terminated_length": 764.0852813720703, "completions/min_length": 341.75, "completions/min_terminated_length": 341.75, "epoch": 0.511388245836756, "grad_norm": 0.3677816689014435, "kl": 1.50537109375, "learning_rate": 1.0534252607261461e-05, "loss": 0.0945, "num_tokens": 843108672.0, "reward": 0.632254496216774, "reward_std": 0.17621283791959286, "rewards/accuracy_reward/mean": 0.14285714458674192, "rewards/accuracy_reward/std": 0.3245743289589882, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.047861308325082064, "step": 1712 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27232142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 835.232177734375, "completions/mean_terminated_length": 767.0381774902344, "completions/min_length": 305.75, "completions/min_terminated_length": 305.75, "epoch": 0.5116869539242775, "grad_norm": 0.29999735951423645, "kl": 1.03515625, "learning_rate": 1.0523359562429441e-05, "loss": 0.0604, "num_tokens": 843555832.0, "reward": 0.6194196790456772, "reward_std": 0.14738418255001307, "rewards/accuracy_reward/mean": 0.12500000186264515, "rewards/accuracy_reward/std": 0.2727537602186203, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03161557391285896, "step": 1713 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 854.8928833007812, "completions/mean_terminated_length": 778.6503753662109, "completions/min_length": 294.25, "completions/min_terminated_length": 294.25, "epoch": 0.511985662011799, "grad_norm": 0.6000915169715881, "kl": 1.2568359375, "learning_rate": 1.0512465894845762e-05, "loss": 0.0909, "num_tokens": 844011560.0, "reward": 0.737723246216774, "reward_std": 0.17999356612563133, "rewards/accuracy_reward/mean": 0.2477678582072258, "rewards/accuracy_reward/std": 0.4207533970475197, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047143861185759306, "step": 1714 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 811.435302734375, "completions/mean_terminated_length": 748.4937591552734, "completions/min_length": 288.5, "completions/min_terminated_length": 288.5, "epoch": 0.5122843700993205, "grad_norm": 0.4008708596229553, "kl": 1.05078125, "learning_rate": 1.0501571617472934e-05, "loss": 0.0586, "num_tokens": 844447659.0, "reward": 0.7689732611179352, "reward_std": 0.1702562691643834, "rewards/accuracy_reward/mean": 0.2767857164144516, "rewards/accuracy_reward/std": 0.44841255247592926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.03658695984631777, "step": 1715 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 847.1027221679688, "completions/mean_terminated_length": 765.6523284912109, "completions/min_length": 363.25, "completions/min_terminated_length": 363.25, "epoch": 0.5125830781868419, "grad_norm": 0.23771914839744568, "kl": 1.6279296875, "learning_rate": 1.0490676743274181e-05, "loss": 0.0876, "num_tokens": 844894073.0, "reward": 0.6796875149011612, "reward_std": 0.1972331702709198, "rewards/accuracy_reward/mean": 0.19196428544819355, "rewards/accuracy_reward/std": 0.3839470446109772, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.04883454320952296, "step": 1716 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 895.6562957763672, "completions/mean_terminated_length": 811.6945495605469, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "epoch": 0.5128817862743634, "grad_norm": 0.30495452880859375, "kl": 1.626953125, "learning_rate": 1.047978128521344e-05, "loss": 0.0786, "num_tokens": 845365855.0, "reward": 0.6227678805589676, "reward_std": 0.13732835510745645, "rewards/accuracy_reward/mean": 0.136160708963871, "rewards/accuracy_reward/std": 0.28134194016456604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05522697977721691, "step": 1717 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 877.9754791259766, "completions/mean_terminated_length": 782.6954650878906, "completions/min_length": 367.75, "completions/min_terminated_length": 367.75, "epoch": 0.5131804943618848, "grad_norm": 0.3884831964969635, "kl": 1.6796875, "learning_rate": 1.0468885256255345e-05, "loss": 0.0829, "num_tokens": 845829044.0, "reward": 0.5954241305589676, "reward_std": 0.12545148096978664, "rewards/accuracy_reward/mean": 0.10491071408614516, "rewards/accuracy_reward/std": 0.27588237076997757, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 1718 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 847.1361846923828, "completions/mean_terminated_length": 744.0214233398438, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 0.5134792024494064, "grad_norm": 0.42014560103416443, "kl": 2.099609375, "learning_rate": 1.045798866936521e-05, "loss": 0.1002, "num_tokens": 846278017.0, "reward": 0.6941964626312256, "reward_std": 0.24001998454332352, "rewards/accuracy_reward/mean": 0.2113095223903656, "rewards/accuracy_reward/std": 0.3929063379764557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 1719 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 883.7299499511719, "completions/mean_terminated_length": 785.0120544433594, "completions/min_length": 336.75, "completions/min_terminated_length": 336.75, "epoch": 0.5137779105369278, "grad_norm": 0.28358718752861023, "kl": 2.185546875, "learning_rate": 1.0447091537509006e-05, "loss": 0.1135, "num_tokens": 846750456.0, "reward": 0.603794664144516, "reward_std": 0.14779604226350784, "rewards/accuracy_reward/mean": 0.11607143050059676, "rewards/accuracy_reward/std": 0.30503665655851364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05248269159346819, "step": 1720 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 892.6674499511719, "completions/mean_terminated_length": 786.1338043212891, "completions/min_length": 333.25, "completions/min_terminated_length": 333.25, "epoch": 0.5140766186244493, "grad_norm": 0.3302272856235504, "kl": 1.904296875, "learning_rate": 1.0436193873653362e-05, "loss": 0.0985, "num_tokens": 847221555.0, "reward": 0.6177455484867096, "reward_std": 0.13088084990158677, "rewards/accuracy_reward/mean": 0.1294642873108387, "rewards/accuracy_reward/std": 0.2712438441812992, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05133028235286474, "step": 1721 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 905.638427734375, "completions/mean_terminated_length": 799.3909149169922, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.5143753267119707, "grad_norm": 0.23275911808013916, "kl": 2.056640625, "learning_rate": 1.0425295690765534e-05, "loss": 0.1136, "num_tokens": 847693697.0, "reward": 0.7031250149011612, "reward_std": 0.1969989687204361, "rewards/accuracy_reward/mean": 0.2142857126891613, "rewards/accuracy_reward/std": 0.39860061556100845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 1722 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 881.7053985595703, "completions/mean_terminated_length": 780.9569244384766, "completions/min_length": 378.75, "completions/min_terminated_length": 378.75, "epoch": 0.5146740347994921, "grad_norm": 0.28175464272499084, "kl": 2.75, "learning_rate": 1.0414397001813396e-05, "loss": 0.1325, "num_tokens": 848156621.0, "reward": 0.6640625149011612, "reward_std": 0.15629338286817074, "rewards/accuracy_reward/mean": 0.17857142724096775, "rewards/accuracy_reward/std": 0.3713728338479996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05846718233078718, "step": 1723 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 917.5714569091797, "completions/mean_terminated_length": 806.6299438476562, "completions/min_length": 459.25, "completions/min_terminated_length": 459.25, "epoch": 0.5149727428870137, "grad_norm": 0.2805110812187195, "kl": 2.5517578125, "learning_rate": 1.0403497819765425e-05, "loss": 0.1268, "num_tokens": 848643693.0, "reward": 0.608816996216774, "reward_std": 0.15687967836856842, "rewards/accuracy_reward/mean": 0.12276785564608872, "rewards/accuracy_reward/std": 0.2989421375095844, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.05568583216518164, "step": 1724 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 931.2857666015625, "completions/mean_terminated_length": 811.3945770263672, "completions/min_length": 436.25, "completions/min_terminated_length": 436.25, "epoch": 0.5152714509745351, "grad_norm": 0.507329523563385, "kl": 3.283203125, "learning_rate": 1.0392598157590687e-05, "loss": 0.151, "num_tokens": 849127997.0, "reward": 0.6372768133878708, "reward_std": 0.1667880229651928, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.3455314002931118, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.060134111903607845, "step": 1725 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 889.5558471679688, "completions/mean_terminated_length": 785.2171325683594, "completions/min_length": 379.5, "completions/min_terminated_length": 379.5, "epoch": 0.5155701590620566, "grad_norm": 0.26173362135887146, "kl": 2.431640625, "learning_rate": 1.0381698028258817e-05, "loss": 0.1217, "num_tokens": 849589318.0, "reward": 0.714285746216774, "reward_std": 0.20477056689560413, "rewards/accuracy_reward/mean": 0.22767857275903225, "rewards/accuracy_reward/std": 0.4060193672776222, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05596293695271015, "step": 1726 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 871.2053833007812, "completions/mean_terminated_length": 746.7720489501953, "completions/min_length": 345.25, "completions/min_terminated_length": 345.25, "epoch": 0.515868867149578, "grad_norm": 0.40004962682724, "kl": 2.78125, "learning_rate": 1.0370797444740008e-05, "loss": 0.1351, "num_tokens": 850053650.0, "reward": 0.5591518059372902, "reward_std": 0.12473234534263611, "rewards/accuracy_reward/mean": 0.0714285708963871, "rewards/accuracy_reward/std": 0.22108019888401031, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 1727 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 882.8772735595703, "completions/mean_terminated_length": 776.3058624267578, "completions/min_length": 404.25, "completions/min_terminated_length": 404.25, "epoch": 0.5161675752370996, "grad_norm": 0.3423154354095459, "kl": 3.1953125, "learning_rate": 1.0359896420004985e-05, "loss": 0.152, "num_tokens": 850518539.0, "reward": 0.6015625298023224, "reward_std": 0.15576180070638657, "rewards/accuracy_reward/mean": 0.11830357182770967, "rewards/accuracy_reward/std": 0.30518436804413795, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06047744210809469, "step": 1728 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 888.0870971679688, "completions/mean_terminated_length": 789.0804443359375, "completions/min_length": 456.0, "completions/min_terminated_length": 456.0, "epoch": 0.516466283324621, "grad_norm": 0.3226085603237152, "kl": 1.517578125, "learning_rate": 1.0348994967025012e-05, "loss": 0.0774, "num_tokens": 850994002.0, "reward": 0.6434152126312256, "reward_std": 0.12213753163814545, "rewards/accuracy_reward/mean": 0.15178571408614516, "rewards/accuracy_reward/std": 0.32793474197387695, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 1729 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 843.0580749511719, "completions/mean_terminated_length": 753.0713806152344, "completions/min_length": 389.5, "completions/min_terminated_length": 389.5, "epoch": 0.5167649914121425, "grad_norm": 0.5715752243995667, "kl": 2.36328125, "learning_rate": 1.033809309877185e-05, "loss": 0.128, "num_tokens": 851437404.0, "reward": 0.7589285969734192, "reward_std": 0.2363758571445942, "rewards/accuracy_reward/mean": 0.2723214291036129, "rewards/accuracy_reward/std": 0.43346797674894333, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.054886242374777794, "step": 1730 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 909.0692443847656, "completions/mean_terminated_length": 791.8819274902344, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.5170636994996639, "grad_norm": 0.3850485384464264, "kl": 3.0546875, "learning_rate": 1.0327190828217763e-05, "loss": 0.157, "num_tokens": 851912091.0, "reward": 0.6958705633878708, "reward_std": 0.20767223089933395, "rewards/accuracy_reward/mean": 0.2265625, "rewards/accuracy_reward/std": 0.4109746143221855, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06403317674994469, "step": 1731 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 912.4933471679688, "completions/mean_terminated_length": 805.0809326171875, "completions/min_length": 407.5, "completions/min_terminated_length": 407.5, "epoch": 0.5173624075871854, "grad_norm": 0.43750685453414917, "kl": 2.703125, "learning_rate": 1.031628816833549e-05, "loss": 0.1168, "num_tokens": 852397832.0, "reward": 0.6835937798023224, "reward_std": 0.21996523067355156, "rewards/accuracy_reward/mean": 0.1997767873108387, "rewards/accuracy_reward/std": 0.3706413060426712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05843239650130272, "step": 1732 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 857.3996124267578, "completions/mean_terminated_length": 745.1274108886719, "completions/min_length": 283.5, "completions/min_terminated_length": 283.5, "epoch": 0.5176611156747069, "grad_norm": 0.5304281711578369, "kl": 2.1015625, "learning_rate": 1.0305385132098229e-05, "loss": 0.103, "num_tokens": 852853499.0, "reward": 0.689732164144516, "reward_std": 0.23604709655046463, "rewards/accuracy_reward/mean": 0.1986607126891613, "rewards/accuracy_reward/std": 0.3904945179820061, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04440389620140195, "step": 1733 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 891.8370971679688, "completions/mean_terminated_length": 784.0586700439453, "completions/min_length": 325.25, "completions/min_terminated_length": 325.25, "epoch": 0.5179598237622284, "grad_norm": 0.3240976333618164, "kl": 2.15625, "learning_rate": 1.0294481732479635e-05, "loss": 0.1064, "num_tokens": 853326674.0, "reward": 0.6395089477300644, "reward_std": 0.1383402869105339, "rewards/accuracy_reward/mean": 0.15178571082651615, "rewards/accuracy_reward/std": 0.3398497626185417, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.053750067949295044, "step": 1734 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49999999999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 902.9375457763672, "completions/mean_terminated_length": 782.0763244628906, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "epoch": 0.5182585318497498, "grad_norm": 0.25652921199798584, "kl": 2.134765625, "learning_rate": 1.0283577982453784e-05, "loss": 0.1038, "num_tokens": 853806102.0, "reward": 0.5831473469734192, "reward_std": 0.10556541476398706, "rewards/accuracy_reward/mean": 0.09747023973613977, "rewards/accuracy_reward/std": 0.2880507819354534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05464820470660925, "step": 1735 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 903.8192291259766, "completions/mean_terminated_length": 793.2847137451172, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "epoch": 0.5185572399372713, "grad_norm": 0.6169731616973877, "kl": 1.537109375, "learning_rate": 1.0272673894995187e-05, "loss": 0.0724, "num_tokens": 854287333.0, "reward": 0.701450914144516, "reward_std": 0.18692532926797867, "rewards/accuracy_reward/mean": 0.2120535708963871, "rewards/accuracy_reward/std": 0.38844969868659973, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 1736 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 935.7946929931641, "completions/mean_terminated_length": 807.6262512207031, "completions/min_length": 404.75, "completions/min_terminated_length": 404.75, "epoch": 0.5188559480247927, "grad_norm": 0.43888697028160095, "kl": 1.646484375, "learning_rate": 1.0261769483078734e-05, "loss": 0.0744, "num_tokens": 854779721.0, "reward": 0.5613839477300644, "reward_std": 0.07987741660326719, "rewards/accuracy_reward/mean": 0.07142857229337096, "rewards/accuracy_reward/std": 0.20885805040597916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04812714271247387, "step": 1737 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5245535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 905.3594055175781, "completions/mean_terminated_length": 775.6152801513672, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.5191546561123143, "grad_norm": 0.44117066264152527, "kl": 1.708984375, "learning_rate": 1.0250864759679715e-05, "loss": 0.0835, "num_tokens": 855257866.0, "reward": 0.5814732313156128, "reward_std": 0.13596962578594685, "rewards/accuracy_reward/mean": 0.09151785913854837, "rewards/accuracy_reward/std": 0.23288775980472565, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048634594306349754, "step": 1738 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 863.6049499511719, "completions/mean_terminated_length": 750.6759185791016, "completions/min_length": 240.5, "completions/min_terminated_length": 240.5, "epoch": 0.5194533641998357, "grad_norm": 0.3866906464099884, "kl": 1.0908203125, "learning_rate": 1.0239959737773791e-05, "loss": 0.046, "num_tokens": 855714297.0, "reward": 0.5920758992433548, "reward_std": 0.12034939229488373, "rewards/accuracy_reward/mean": 0.0959821417927742, "rewards/accuracy_reward/std": 0.2943203002214432, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1739 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 903.7857360839844, "completions/mean_terminated_length": 769.1797180175781, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.5197520722873572, "grad_norm": 0.6955625414848328, "kl": 1.9619140625, "learning_rate": 1.022905443033697e-05, "loss": 0.1096, "num_tokens": 856194201.0, "reward": 0.643415205180645, "reward_std": 0.1501287529245019, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.28527384623885155, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810296744108, "step": 1740 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 857.3192291259766, "completions/mean_terminated_length": 756.6728973388672, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "epoch": 0.5200507803748786, "grad_norm": 0.2502782642841339, "kl": 1.7529296875, "learning_rate": 1.0218148850345613e-05, "loss": 0.1029, "num_tokens": 856646312.0, "reward": 0.7293527126312256, "reward_std": 0.1256052441895008, "rewards/accuracy_reward/mean": 0.2388392835855484, "rewards/accuracy_reward/std": 0.40870529413223267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04517605667933822, "step": 1741 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 902.8817291259766, "completions/mean_terminated_length": 745.8107147216797, "completions/min_length": 342.5, "completions/min_terminated_length": 342.5, "epoch": 0.5203494884624001, "grad_norm": 0.30184540152549744, "kl": 2.626953125, "learning_rate": 1.0207243010776387e-05, "loss": 0.1289, "num_tokens": 857117075.0, "reward": 0.613839328289032, "reward_std": 0.1397339627146721, "rewards/accuracy_reward/mean": 0.12946428591385484, "rewards/accuracy_reward/std": 0.3042755499482155, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.060082768090069294, "step": 1742 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 855.8147735595703, "completions/mean_terminated_length": 710.5628356933594, "completions/min_length": 295.25, "completions/min_terminated_length": 295.25, "epoch": 0.5206481965499216, "grad_norm": 0.28031080961227417, "kl": 1.166015625, "learning_rate": 1.0196336924606282e-05, "loss": 0.0766, "num_tokens": 857573072.0, "reward": 0.6300223618745804, "reward_std": 0.15379897877573967, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.32475487142801285, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 1743 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 938.4375457763672, "completions/mean_terminated_length": 817.9763336181641, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.5209469046374431, "grad_norm": 0.2894590198993683, "kl": 1.712890625, "learning_rate": 1.0185430604812581e-05, "loss": 0.0716, "num_tokens": 858069668.0, "reward": 0.6808035969734192, "reward_std": 0.191398773342371, "rewards/accuracy_reward/mean": 0.1897321492433548, "rewards/accuracy_reward/std": 0.38760605454444885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044658167753368616, "step": 1744 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 912.6719055175781, "completions/mean_terminated_length": 808.0850524902344, "completions/min_length": 438.75, "completions/min_terminated_length": 438.75, "epoch": 0.5212456127249645, "grad_norm": 0.30718424916267395, "kl": 1.421875, "learning_rate": 1.0174524064372837e-05, "loss": 0.0735, "num_tokens": 858546641.0, "reward": 0.6562500298023224, "reward_std": 0.14544845186173916, "rewards/accuracy_reward/mean": 0.16294642724096775, "rewards/accuracy_reward/std": 0.3546401113271713, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 1745 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 914.1362152099609, "completions/mean_terminated_length": 785.3923950195312, "completions/min_length": 276.5, "completions/min_terminated_length": 276.5, "epoch": 0.521544320812486, "grad_norm": 0.1910678595304489, "kl": 1.7958984375, "learning_rate": 1.0163617316264869e-05, "loss": 0.0774, "num_tokens": 859027502.0, "reward": 0.602120578289032, "reward_std": 0.11456112843006849, "rewards/accuracy_reward/mean": 0.11235119169577956, "rewards/accuracy_reward/std": 0.29940190538764, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241046637297, "step": 1746 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 902.2187805175781, "completions/mean_terminated_length": 768.0744018554688, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "epoch": 0.5218430289000074, "grad_norm": 0.14701955020427704, "kl": 1.191650390625, "learning_rate": 1.0152710373466746e-05, "loss": 0.0505, "num_tokens": 859503296.0, "reward": 0.6205357313156128, "reward_std": 0.12397184409201145, "rewards/accuracy_reward/mean": 0.12723214155994356, "rewards/accuracy_reward/std": 0.265099935233593, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.034245037473738194, "step": 1747 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 928.7076416015625, "completions/mean_terminated_length": 786.1041107177734, "completions/min_length": 418.5, "completions/min_terminated_length": 418.5, "epoch": 0.522141736987529, "grad_norm": 0.1850672960281372, "kl": 1.544921875, "learning_rate": 1.0141803248956768e-05, "loss": 0.0855, "num_tokens": 860003565.0, "reward": 0.6328125149011612, "reward_std": 0.16697768727317452, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.282431460916996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04272336792200804, "step": 1748 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 890.0223541259766, "completions/mean_terminated_length": 774.2213134765625, "completions/min_length": 347.75, "completions/min_terminated_length": 347.75, "epoch": 0.5224404450750504, "grad_norm": 0.2291986644268036, "kl": 1.12890625, "learning_rate": 1.0130895955713445e-05, "loss": 0.0596, "num_tokens": 860465879.0, "reward": 0.7555803805589676, "reward_std": 0.15435395948588848, "rewards/accuracy_reward/mean": 0.2611607138533145, "rewards/accuracy_reward/std": 0.3743425440043211, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1749 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 877.7254943847656, "completions/mean_terminated_length": 745.9166107177734, "completions/min_length": 311.5, "completions/min_terminated_length": 311.5, "epoch": 0.5227391531625719, "grad_norm": 0.41884714365005493, "kl": 1.689453125, "learning_rate": 1.0119988506715497e-05, "loss": 0.0795, "num_tokens": 860932892.0, "reward": 0.666294664144516, "reward_std": 0.1590942470356822, "rewards/accuracy_reward/mean": 0.17410714668221772, "rewards/accuracy_reward/std": 0.32727085426449776, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 1750 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5803571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 934.5134429931641, "completions/mean_terminated_length": 825.1288604736328, "completions/min_length": 359.5, "completions/min_terminated_length": 359.5, "epoch": 0.5230378612500933, "grad_norm": 0.19768713414669037, "kl": 1.4990234375, "learning_rate": 1.0109080914941825e-05, "loss": 0.076, "num_tokens": 861423522.0, "reward": 0.568638414144516, "reward_std": 0.13813269510865211, "rewards/accuracy_reward/mean": 0.07589285587891936, "rewards/accuracy_reward/std": 0.25006433203816414, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 1751 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 947.9866485595703, "completions/mean_terminated_length": 825.5769500732422, "completions/min_length": 448.5, "completions/min_terminated_length": 448.5, "epoch": 0.5233365693376149, "grad_norm": 0.2258322536945343, "kl": 1.94921875, "learning_rate": 1.0098173193371498e-05, "loss": 0.0865, "num_tokens": 861918844.0, "reward": 0.6780134290456772, "reward_std": 0.17923789843916893, "rewards/accuracy_reward/mean": 0.1897321455180645, "rewards/accuracy_reward/std": 0.3757617920637131, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 1752 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6049107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 933.2612152099609, "completions/mean_terminated_length": 798.3697204589844, "completions/min_length": 415.75, "completions/min_terminated_length": 415.75, "epoch": 0.5236352774251363, "grad_norm": 0.21566922962665558, "kl": 1.654296875, "learning_rate": 1.008726535498374e-05, "loss": 0.0759, "num_tokens": 862418241.0, "reward": 0.7321428805589676, "reward_std": 0.18995241448283195, "rewards/accuracy_reward/mean": 0.2410714253783226, "rewards/accuracy_reward/std": 0.41811148077249527, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04620361328125, "step": 1753 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5736607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 913.200927734375, "completions/mean_terminated_length": 776.9974060058594, "completions/min_length": 382.75, "completions/min_terminated_length": 382.75, "epoch": 0.5239339855126578, "grad_norm": 0.30998846888542175, "kl": 1.6796875, "learning_rate": 1.0076357412757918e-05, "loss": 0.0817, "num_tokens": 862909051.0, "reward": 0.6311384290456772, "reward_std": 0.12883305549621582, "rewards/accuracy_reward/mean": 0.14062500232830644, "rewards/accuracy_reward/std": 0.3125814311206341, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1754 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 910.5870971679688, "completions/mean_terminated_length": 792.3928375244141, "completions/min_length": 435.5, "completions/min_terminated_length": 435.5, "epoch": 0.5242326936001792, "grad_norm": 0.17703931033611298, "kl": 1.353515625, "learning_rate": 1.0065449379673519e-05, "loss": 0.0671, "num_tokens": 863386338.0, "reward": 0.6378348469734192, "reward_std": 0.15722043253481388, "rewards/accuracy_reward/mean": 0.14508928591385484, "rewards/accuracy_reward/std": 0.298670195043087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818479284644, "step": 1755 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44642857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 889.3326263427734, "completions/mean_terminated_length": 786.6479949951172, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "epoch": 0.5245314016877007, "grad_norm": 0.2605804204940796, "kl": 1.587890625, "learning_rate": 1.0054541268710139e-05, "loss": 0.0863, "num_tokens": 863862167.0, "reward": 0.6501116305589676, "reward_std": 0.13455823250114918, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.3651215210556984, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.043475935235619545, "step": 1756 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 926.6094207763672, "completions/mean_terminated_length": 799.3812255859375, "completions/min_length": 459.75, "completions/min_terminated_length": 459.75, "epoch": 0.5248301097752222, "grad_norm": 0.1508232206106186, "kl": 0.9482421875, "learning_rate": 1.0043633092847468e-05, "loss": 0.0514, "num_tokens": 864350424.0, "reward": 0.7008928954601288, "reward_std": 0.13356024399399757, "rewards/accuracy_reward/mean": 0.2053571417927742, "rewards/accuracy_reward/std": 0.3851066455245018, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1757 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 903.2902221679688, "completions/mean_terminated_length": 796.8676300048828, "completions/min_length": 397.75, "completions/min_terminated_length": 397.75, "epoch": 0.5251288178627437, "grad_norm": 0.2561376094818115, "kl": 1.115234375, "learning_rate": 1.003272486506527e-05, "loss": 0.0639, "num_tokens": 864827018.0, "reward": 0.6361607387661934, "reward_std": 0.1426575342193246, "rewards/accuracy_reward/mean": 0.14285714295692742, "rewards/accuracy_reward/std": 0.24931377731263638, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 1758 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5602678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 922.0714721679688, "completions/mean_terminated_length": 800.0883483886719, "completions/min_length": 488.5, "completions/min_terminated_length": 488.5, "epoch": 0.5254275259502651, "grad_norm": 0.7564290761947632, "kl": 1.4326171875, "learning_rate": 1.002181659834337e-05, "loss": 0.0719, "num_tokens": 865311610.0, "reward": 0.666294664144516, "reward_std": 0.14423455856740475, "rewards/accuracy_reward/mean": 0.1763392868451774, "rewards/accuracy_reward/std": 0.34364550560712814, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764320462942, "step": 1759 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 908.2187957763672, "completions/mean_terminated_length": 796.7170104980469, "completions/min_length": 324.25, "completions/min_terminated_length": 324.25, "epoch": 0.5257262340377866, "grad_norm": 0.16696567833423615, "kl": 1.43359375, "learning_rate": 1.0010908305661644e-05, "loss": 0.0642, "num_tokens": 865791676.0, "reward": 0.6958705633878708, "reward_std": 0.19111331924796104, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.39603830873966217, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 1760 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 937.5580749511719, "completions/mean_terminated_length": 822.1830902099609, "completions/min_length": 453.5, "completions/min_terminated_length": 453.5, "epoch": 0.526024942125308, "grad_norm": 0.18149806559085846, "kl": 1.810546875, "learning_rate": 1e-05, "loss": 0.0994, "num_tokens": 866279366.0, "reward": 0.6841517984867096, "reward_std": 0.20160535164177418, "rewards/accuracy_reward/mean": 0.20163690485060215, "rewards/accuracy_reward/std": 0.3629226014018059, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 1761 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 919.9241485595703, "completions/mean_terminated_length": 809.2567901611328, "completions/min_length": 420.25, "completions/min_terminated_length": 420.25, "epoch": 0.5263236502128296, "grad_norm": 0.17377281188964844, "kl": 1.06884765625, "learning_rate": 9.989091694338356e-06, "loss": 0.0565, "num_tokens": 866759396.0, "reward": 0.6406250298023224, "reward_std": 0.12253673281520605, "rewards/accuracy_reward/mean": 0.1473214253783226, "rewards/accuracy_reward/std": 0.34965961426496506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.033647436648607254, "step": 1762 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 944.2835235595703, "completions/mean_terminated_length": 808.5743103027344, "completions/min_length": 443.0, "completions/min_terminated_length": 443.0, "epoch": 0.526622358300351, "grad_norm": 0.2844134271144867, "kl": 2.19140625, "learning_rate": 9.978183401656632e-06, "loss": 0.1266, "num_tokens": 867269507.0, "reward": 0.6478795111179352, "reward_std": 0.20683376491069794, "rewards/accuracy_reward/mean": 0.16071428265422583, "rewards/accuracy_reward/std": 0.35303011536598206, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05466675665229559, "step": 1763 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 919.2611999511719, "completions/mean_terminated_length": 803.6666412353516, "completions/min_length": 453.25, "completions/min_terminated_length": 453.25, "epoch": 0.5269210663878725, "grad_norm": 0.30729174613952637, "kl": 2.6171875, "learning_rate": 9.967275134934732e-06, "loss": 0.1324, "num_tokens": 867753720.0, "reward": 0.550223246216774, "reward_std": 0.1308201653882861, "rewards/accuracy_reward/mean": 0.06696428405120969, "rewards/accuracy_reward/std": 0.19577499106526375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06137341819703579, "step": 1764 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 929.3750457763672, "completions/mean_terminated_length": 823.7146148681641, "completions/min_length": 460.25, "completions/min_terminated_length": 460.25, "epoch": 0.5272197744753939, "grad_norm": 0.21261590719223022, "kl": 1.7802734375, "learning_rate": 9.956366907152536e-06, "loss": 0.0844, "num_tokens": 868246800.0, "reward": 0.5965401977300644, "reward_std": 0.11458348296582699, "rewards/accuracy_reward/mean": 0.10714285750873387, "rewards/accuracy_reward/std": 0.28342034481465816, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810296744108, "step": 1765 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 899.9844207763672, "completions/mean_terminated_length": 797.0719909667969, "completions/min_length": 503.5, "completions/min_terminated_length": 503.5, "epoch": 0.5275184825629153, "grad_norm": 1.0575830936431885, "kl": 1.9375, "learning_rate": 9.945458731289863e-06, "loss": 0.11, "num_tokens": 868721753.0, "reward": 0.6210937798023224, "reward_std": 0.15638107433915138, "rewards/accuracy_reward/mean": 0.13169642630964518, "rewards/accuracy_reward/std": 0.3181811720132828, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050148884765803814, "step": 1766 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6026785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 927.5603179931641, "completions/mean_terminated_length": 796.8231658935547, "completions/min_length": 346.25, "completions/min_terminated_length": 346.25, "epoch": 0.5278171906504369, "grad_norm": 0.34552332758903503, "kl": 2.193359375, "learning_rate": 9.934550620326483e-06, "loss": 0.1041, "num_tokens": 869213444.0, "reward": 0.511160746216774, "reward_std": 0.09650817699730396, "rewards/accuracy_reward/mean": 0.026339286006987095, "rewards/accuracy_reward/std": 0.13636856898665428, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05448929313570261, "step": 1767 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6138392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 951.2768402099609, "completions/mean_terminated_length": 836.5696563720703, "completions/min_length": 420.5, "completions/min_terminated_length": 420.5, "epoch": 0.5281158987379583, "grad_norm": 0.2252953201532364, "kl": 2.099609375, "learning_rate": 9.923642587242082e-06, "loss": 0.0957, "num_tokens": 869713584.0, "reward": 0.562500037252903, "reward_std": 0.12928710971027613, "rewards/accuracy_reward/mean": 0.07589285913854837, "rewards/accuracy_reward/std": 0.2220916822552681, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05632450245320797, "step": 1768 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5401785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 913.6741485595703, "completions/mean_terminated_length": 799.8857269287109, "completions/min_length": 454.75, "completions/min_terminated_length": 454.75, "epoch": 0.5284146068254798, "grad_norm": 0.3088627755641937, "kl": 2.623046875, "learning_rate": 9.912734645016262e-06, "loss": 0.1334, "num_tokens": 870201582.0, "reward": 0.5770089626312256, "reward_std": 0.15800689905881882, "rewards/accuracy_reward/mean": 0.09374999976716936, "rewards/accuracy_reward/std": 0.24035475589334965, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06003319285809994, "step": 1769 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 904.5714874267578, "completions/mean_terminated_length": 788.868408203125, "completions/min_length": 472.25, "completions/min_terminated_length": 472.25, "epoch": 0.5287133149130012, "grad_norm": 0.3907054364681244, "kl": 2.22265625, "learning_rate": 9.901826806628505e-06, "loss": 0.1108, "num_tokens": 870676782.0, "reward": 0.6718750298023224, "reward_std": 0.14966992661356926, "rewards/accuracy_reward/mean": 0.18526785634458065, "rewards/accuracy_reward/std": 0.3786718547344208, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05548384319990873, "step": 1770 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 901.6205749511719, "completions/mean_terminated_length": 756.1088104248047, "completions/min_length": 414.25, "completions/min_terminated_length": 414.25, "epoch": 0.5290120230005227, "grad_norm": 0.2672457993030548, "kl": 2.580078125, "learning_rate": 9.890919085058179e-06, "loss": 0.1271, "num_tokens": 871158756.0, "reward": 0.6277902126312256, "reward_std": 0.1397693231701851, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.2953105494379997, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05868698377162218, "step": 1771 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 904.7545166015625, "completions/mean_terminated_length": 783.3861541748047, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "epoch": 0.5293107310880442, "grad_norm": 0.32938405871391296, "kl": 2.240234375, "learning_rate": 9.880011493284504e-06, "loss": 0.1158, "num_tokens": 871643702.0, "reward": 0.5870535969734192, "reward_std": 0.11511478573083878, "rewards/accuracy_reward/mean": 0.10565476189367473, "rewards/accuracy_reward/std": 0.26240901462733746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 1772 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 882.6272735595703, "completions/mean_terminated_length": 798.9325561523438, "completions/min_length": 411.5, "completions/min_terminated_length": 411.5, "epoch": 0.5296094391755657, "grad_norm": 0.28516995906829834, "kl": 1.64453125, "learning_rate": 9.869104044286558e-06, "loss": 0.0937, "num_tokens": 872111119.0, "reward": 0.7617187947034836, "reward_std": 0.24871626868844032, "rewards/accuracy_reward/mean": 0.2700892901048064, "rewards/accuracy_reward/std": 0.39538655430078506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1773 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 877.6585235595703, "completions/mean_terminated_length": 777.8174743652344, "completions/min_length": 396.0, "completions/min_terminated_length": 396.0, "epoch": 0.5299081472630871, "grad_norm": 0.26753196120262146, "kl": 2.126953125, "learning_rate": 9.858196751043232e-06, "loss": 0.1054, "num_tokens": 872575830.0, "reward": 0.6780134290456772, "reward_std": 0.22051437012851238, "rewards/accuracy_reward/mean": 0.18973213993012905, "rewards/accuracy_reward/std": 0.37958522140979767, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05223577655851841, "step": 1774 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 912.872802734375, "completions/mean_terminated_length": 791.8198547363281, "completions/min_length": 361.75, "completions/min_terminated_length": 361.75, "epoch": 0.5302068553506086, "grad_norm": 0.33158066868782043, "kl": 2.671875, "learning_rate": 9.847289626533257e-06, "loss": 0.1252, "num_tokens": 873061661.0, "reward": 0.6082589626312256, "reward_std": 0.19189694058150053, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.27599186450242996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06169993244111538, "step": 1775 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 902.8973541259766, "completions/mean_terminated_length": 776.1119995117188, "completions/min_length": 416.75, "completions/min_terminated_length": 416.75, "epoch": 0.53050556343813, "grad_norm": 0.7331908941268921, "kl": 2.73046875, "learning_rate": 9.836382683735133e-06, "loss": 0.1304, "num_tokens": 873540383.0, "reward": 0.5770089477300644, "reward_std": 0.11691267229616642, "rewards/accuracy_reward/mean": 0.09747023927047849, "rewards/accuracy_reward/std": 0.28095315769314766, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06256846059113741, "step": 1776 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 875.9643096923828, "completions/mean_terminated_length": 768.0997619628906, "completions/min_length": 391.75, "completions/min_terminated_length": 391.75, "epoch": 0.5308042715256516, "grad_norm": 0.2577129900455475, "kl": 2.814453125, "learning_rate": 9.825475935627165e-06, "loss": 0.1534, "num_tokens": 874002303.0, "reward": 0.633370578289032, "reward_std": 0.15941356867551804, "rewards/accuracy_reward/mean": 0.14955357229337096, "rewards/accuracy_reward/std": 0.32591281831264496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06117267720401287, "step": 1777 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 872.9487152099609, "completions/mean_terminated_length": 754.6194000244141, "completions/min_length": 391.75, "completions/min_terminated_length": 391.75, "epoch": 0.531102979613173, "grad_norm": 0.2861352264881134, "kl": 2.171875, "learning_rate": 9.81456939518742e-06, "loss": 0.1129, "num_tokens": 874463784.0, "reward": 0.6473214626312256, "reward_std": 0.137859757989645, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.35402341187000275, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 1778 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 875.3862152099609, "completions/mean_terminated_length": 772.9833374023438, "completions/min_length": 437.5, "completions/min_terminated_length": 437.5, "epoch": 0.5314016877006945, "grad_norm": 0.4082801938056946, "kl": 2.775390625, "learning_rate": 9.80366307539372e-06, "loss": 0.16, "num_tokens": 874931605.0, "reward": 0.6272321715950966, "reward_std": 0.21159970201551914, "rewards/accuracy_reward/mean": 0.14508928847499192, "rewards/accuracy_reward/std": 0.31732177548110485, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06391280237585306, "step": 1779 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5758928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 912.8594055175781, "completions/mean_terminated_length": 772.1390838623047, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.5317003957882159, "grad_norm": 0.4408363401889801, "kl": 3.03515625, "learning_rate": 9.792756989223614e-06, "loss": 0.1377, "num_tokens": 875415958.0, "reward": 0.6607143133878708, "reward_std": 0.1869259998202324, "rewards/accuracy_reward/mean": 0.17857142724096775, "rewards/accuracy_reward/std": 0.37890487909317017, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4821428582072258, "rewards/tag_count_reward/std": 0.06337972916662693, "step": 1780 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 870.5134429931641, "completions/mean_terminated_length": 719.0885467529297, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "epoch": 0.5319991038757375, "grad_norm": 0.24939413368701935, "kl": 2.8125, "learning_rate": 9.78185114965439e-06, "loss": 0.1509, "num_tokens": 875875356.0, "reward": 0.6992187798023224, "reward_std": 0.1844206228852272, "rewards/accuracy_reward/mean": 0.21428571082651615, "rewards/accuracy_reward/std": 0.38707824796438217, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05902999825775623, "step": 1781 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 883.857177734375, "completions/mean_terminated_length": 762.4270782470703, "completions/min_length": 427.75, "completions/min_terminated_length": 427.75, "epoch": 0.5322978119632589, "grad_norm": 0.2288314551115036, "kl": 2.955078125, "learning_rate": 9.770945569663028e-06, "loss": 0.1447, "num_tokens": 876343180.0, "reward": 0.6568080633878708, "reward_std": 0.15988698042929173, "rewards/accuracy_reward/mean": 0.1741071455180645, "rewards/accuracy_reward/std": 0.3793282210826874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06313127744942904, "step": 1782 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39732142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 871.4308624267578, "completions/mean_terminated_length": 772.5853118896484, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.5325965200507804, "grad_norm": 0.37694835662841797, "kl": 2.455078125, "learning_rate": 9.760040262226214e-06, "loss": 0.127, "num_tokens": 876807805.0, "reward": 0.6406250149011612, "reward_std": 0.1291141826659441, "rewards/accuracy_reward/mean": 0.15178571199066937, "rewards/accuracy_reward/std": 0.31471060775220394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 1783 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 872.6161193847656, "completions/mean_terminated_length": 749.1147308349609, "completions/min_length": 375.5, "completions/min_terminated_length": 375.5, "epoch": 0.5328952281383018, "grad_norm": 0.2711922228336334, "kl": 2.248046875, "learning_rate": 9.749135240320288e-06, "loss": 0.105, "num_tokens": 877283857.0, "reward": 0.6021205633878708, "reward_std": 0.16009843721985817, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.30233847722411156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 1784 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 854.5379791259766, "completions/mean_terminated_length": 761.0987091064453, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "epoch": 0.5331939362258233, "grad_norm": 0.30404654145240784, "kl": 1.97265625, "learning_rate": 9.738230516921272e-06, "loss": 0.1118, "num_tokens": 877735682.0, "reward": 0.7717634290456772, "reward_std": 0.2469371110200882, "rewards/accuracy_reward/mean": 0.292410708963871, "rewards/accuracy_reward/std": 0.45644866675138474, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 1785 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 833.9375305175781, "completions/mean_terminated_length": 702.20263671875, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "epoch": 0.5334926443133448, "grad_norm": 0.36222314834594727, "kl": 2.2109375, "learning_rate": 9.727326105004818e-06, "loss": 0.1167, "num_tokens": 878183222.0, "reward": 0.641183078289032, "reward_std": 0.1681799329817295, "rewards/accuracy_reward/mean": 0.15178571408614516, "rewards/accuracy_reward/std": 0.32623008638620377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05040315631777048, "step": 1786 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 849.8192291259766, "completions/mean_terminated_length": 754.3794860839844, "completions/min_length": 412.25, "completions/min_terminated_length": 412.25, "epoch": 0.5337913524008663, "grad_norm": 0.28748491406440735, "kl": 1.802734375, "learning_rate": 9.716422017546219e-06, "loss": 0.0993, "num_tokens": 878633045.0, "reward": 0.6741071790456772, "reward_std": 0.13620953261852264, "rewards/accuracy_reward/mean": 0.1975446455180645, "rewards/accuracy_reward/std": 0.39340925961732864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03992978110909462, "step": 1787 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39062499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 867.4196929931641, "completions/mean_terminated_length": 773.3024749755859, "completions/min_length": 366.25, "completions/min_terminated_length": 366.25, "epoch": 0.5340900604883877, "grad_norm": 0.265895813703537, "kl": 1.890625, "learning_rate": 9.705518267520369e-06, "loss": 0.1019, "num_tokens": 879105345.0, "reward": 0.6835937798023224, "reward_std": 0.14951907098293304, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.3932162746787071, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04334343643859029, "step": 1788 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2767857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 807.0357360839844, "completions/mean_terminated_length": 725.9331970214844, "completions/min_length": 322.75, "completions/min_terminated_length": 322.75, "epoch": 0.5343887685759092, "grad_norm": 0.3833935856819153, "kl": 1.8115234375, "learning_rate": 9.694614867901776e-06, "loss": 0.1077, "num_tokens": 879539313.0, "reward": 0.6138393133878708, "reward_std": 0.11822825483977795, "rewards/accuracy_reward/mean": 0.12276785681024194, "rewards/accuracy_reward/std": 0.3026667796075344, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 1789 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 829.2656555175781, "completions/mean_terminated_length": 742.9692993164062, "completions/min_length": 395.25, "completions/min_terminated_length": 395.25, "epoch": 0.5346874766634306, "grad_norm": 0.2420881688594818, "kl": 2.205078125, "learning_rate": 9.683711831664516e-06, "loss": 0.1125, "num_tokens": 879983960.0, "reward": 0.6540178954601288, "reward_std": 0.15752246975898743, "rewards/accuracy_reward/mean": 0.1651785746216774, "rewards/accuracy_reward/std": 0.3063866198062897, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050666457042098045, "step": 1790 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 847.8125305175781, "completions/mean_terminated_length": 743.5469207763672, "completions/min_length": 316.5, "completions/min_terminated_length": 316.5, "epoch": 0.5349861847509522, "grad_norm": 0.20833317935466766, "kl": 2.037109375, "learning_rate": 9.67280917178224e-06, "loss": 0.1074, "num_tokens": 880436036.0, "reward": 0.5948660969734192, "reward_std": 0.15470791794359684, "rewards/accuracy_reward/mean": 0.1049107126891613, "rewards/accuracy_reward/std": 0.30537374317646027, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04618143197149038, "step": 1791 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 857.3638916015625, "completions/mean_terminated_length": 772.6715240478516, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "epoch": 0.5352848928384736, "grad_norm": 0.23795853555202484, "kl": 1.763671875, "learning_rate": 9.661906901228153e-06, "loss": 0.0893, "num_tokens": 880892327.0, "reward": 0.710379496216774, "reward_std": 0.20828397385776043, "rewards/accuracy_reward/mean": 0.22098214365541935, "rewards/accuracy_reward/std": 0.38506052643060684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 1792 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 823.013427734375, "completions/mean_terminated_length": 723.7939758300781, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 0.5355836009259951, "grad_norm": 0.26826298236846924, "kl": 1.677734375, "learning_rate": 9.651005032974994e-06, "loss": 0.0733, "num_tokens": 881325805.0, "reward": 0.7059152275323868, "reward_std": 0.1744200848042965, "rewards/accuracy_reward/mean": 0.21428571082651615, "rewards/accuracy_reward/std": 0.396680124104023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 1793 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2946428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 845.091552734375, "completions/mean_terminated_length": 776.5740966796875, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.5358823090135165, "grad_norm": 0.24079547822475433, "kl": 1.0859375, "learning_rate": 9.640103579995019e-06, "loss": 0.0496, "num_tokens": 881778038.0, "reward": 0.6986607313156128, "reward_std": 0.15210316516458988, "rewards/accuracy_reward/mean": 0.20535714831203222, "rewards/accuracy_reward/std": 0.34905217587947845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.039343451615422964, "step": 1794 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 865.3326263427734, "completions/mean_terminated_length": 761.3424987792969, "completions/min_length": 376.25, "completions/min_terminated_length": 376.25, "epoch": 0.536181017101038, "grad_norm": 0.45959556102752686, "kl": 1.85546875, "learning_rate": 9.629202555259997e-06, "loss": 0.097, "num_tokens": 882239451.0, "reward": 0.7438616454601288, "reward_std": 0.16756609454751015, "rewards/accuracy_reward/mean": 0.2544642901048064, "rewards/accuracy_reward/std": 0.3854062631726265, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 1795 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.25223214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 819.8973541259766, "completions/mean_terminated_length": 750.3799591064453, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "epoch": 0.5364797251885595, "grad_norm": 0.21080011129379272, "kl": 1.423828125, "learning_rate": 9.618301971741185e-06, "loss": 0.0898, "num_tokens": 882679549.0, "reward": 0.6986607611179352, "reward_std": 0.22093185037374496, "rewards/accuracy_reward/mean": 0.2053571450524032, "rewards/accuracy_reward/std": 0.3684878721833229, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1796 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 828.9062805175781, "completions/mean_terminated_length": 741.7728271484375, "completions/min_length": 373.25, "completions/min_terminated_length": 373.25, "epoch": 0.536778433276081, "grad_norm": 0.17423675954341888, "kl": 0.9296875, "learning_rate": 9.607401842409318e-06, "loss": 0.0444, "num_tokens": 883118707.0, "reward": 0.6919643133878708, "reward_std": 0.18301912397146225, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.3856573924422264, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.032084173522889614, "step": 1797 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 855.122802734375, "completions/mean_terminated_length": 755.9533081054688, "completions/min_length": 281.5, "completions/min_terminated_length": 281.5, "epoch": 0.5370771413636024, "grad_norm": 0.2874497175216675, "kl": 1.24755859375, "learning_rate": 9.596502180234578e-06, "loss": 0.0647, "num_tokens": 883565914.0, "reward": 0.595982164144516, "reward_std": 0.10161808505654335, "rewards/accuracy_reward/mean": 0.10044642724096775, "rewards/accuracy_reward/std": 0.29763199761509895, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.028279099613428116, "step": 1798 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 846.8705749511719, "completions/mean_terminated_length": 754.5267181396484, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "epoch": 0.5373758494511239, "grad_norm": 0.31943151354789734, "kl": 2.15625, "learning_rate": 9.58560299818661e-06, "loss": 0.1172, "num_tokens": 884019600.0, "reward": 0.6534598469734192, "reward_std": 0.16061478108167648, "rewards/accuracy_reward/mean": 0.16517857206054032, "rewards/accuracy_reward/std": 0.3293354567140341, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918293237686, "step": 1799 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 849.154052734375, "completions/mean_terminated_length": 763.7482452392578, "completions/min_length": 467.5, "completions/min_terminated_length": 467.5, "epoch": 0.5376745575386453, "grad_norm": 0.2542516589164734, "kl": 1.94140625, "learning_rate": 9.574704309234471e-06, "loss": 0.1032, "num_tokens": 884471941.0, "reward": 0.6277902126312256, "reward_std": 0.13107513822615147, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3317105323076248, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04884427320212126, "step": 1800 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 870.5156555175781, "completions/mean_terminated_length": 782.9787902832031, "completions/min_length": 417.5, "completions/min_terminated_length": 417.5, "epoch": 0.5379732656261669, "grad_norm": 0.2730584442615509, "kl": 1.9765625, "learning_rate": 9.563806126346643e-06, "loss": 0.0987, "num_tokens": 884939100.0, "reward": 0.6757812798023224, "reward_std": 0.16709664836525917, "rewards/accuracy_reward/mean": 0.18749999813735485, "rewards/accuracy_reward/std": 0.37204983085393906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 1801 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 881.8237152099609, "completions/mean_terminated_length": 783.3501281738281, "completions/min_length": 443.25, "completions/min_terminated_length": 443.25, "epoch": 0.5382719737136883, "grad_norm": 0.3797371983528137, "kl": 1.79296875, "learning_rate": 9.552908462490995e-06, "loss": 0.092, "num_tokens": 885398717.0, "reward": 0.6088169887661934, "reward_std": 0.12841391563415527, "rewards/accuracy_reward/mean": 0.11830357275903225, "rewards/accuracy_reward/std": 0.2658891901373863, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04597289999946952, "step": 1802 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 855.5759429931641, "completions/mean_terminated_length": 760.20947265625, "completions/min_length": 314.5, "completions/min_terminated_length": 314.5, "epoch": 0.5385706818012098, "grad_norm": 0.31499236822128296, "kl": 1.91796875, "learning_rate": 9.542011330634796e-06, "loss": 0.1134, "num_tokens": 885855183.0, "reward": 0.6635044813156128, "reward_std": 0.17207206599414349, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.35086266696453094, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 1803 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 864.5647735595703, "completions/mean_terminated_length": 776.8205108642578, "completions/min_length": 368.25, "completions/min_terminated_length": 368.25, "epoch": 0.5388693898887312, "grad_norm": 0.34695279598236084, "kl": 1.7138671875, "learning_rate": 9.531114743744658e-06, "loss": 0.085, "num_tokens": 886317100.0, "reward": 0.6852678954601288, "reward_std": 0.1544001423753798, "rewards/accuracy_reward/mean": 0.19196428544819355, "rewards/accuracy_reward/std": 0.30227256566286087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 1804 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 859.2500305175781, "completions/mean_terminated_length": 756.0316619873047, "completions/min_length": 384.25, "completions/min_terminated_length": 384.25, "epoch": 0.5391680979762528, "grad_norm": 0.281429260969162, "kl": 2.3671875, "learning_rate": 9.520218714786564e-06, "loss": 0.1177, "num_tokens": 886781388.0, "reward": 0.5262277126312256, "reward_std": 0.08042575418949127, "rewards/accuracy_reward/mean": 0.035714286379516125, "rewards/accuracy_reward/std": 0.15270046889781952, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 1805 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 899.0491485595703, "completions/mean_terminated_length": 798.0274810791016, "completions/min_length": 394.25, "completions/min_terminated_length": 394.25, "epoch": 0.5394668060637742, "grad_norm": 0.3322172164916992, "kl": 2.693359375, "learning_rate": 9.50932325672582e-06, "loss": 0.1376, "num_tokens": 887261938.0, "reward": 0.6160714626312256, "reward_std": 0.16174946911633015, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.32441527396440506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050612835213541985, "step": 1806 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 868.7076110839844, "completions/mean_terminated_length": 738.8934631347656, "completions/min_length": 368.25, "completions/min_terminated_length": 368.25, "epoch": 0.5397655141512957, "grad_norm": 0.6992748379707336, "kl": 2.619140625, "learning_rate": 9.498428382527066e-06, "loss": 0.1273, "num_tokens": 887729663.0, "reward": 0.7059152126312256, "reward_std": 0.19048713333904743, "rewards/accuracy_reward/mean": 0.2142857164144516, "rewards/accuracy_reward/std": 0.40038619190454483, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04288960574194789, "step": 1807 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 862.122802734375, "completions/mean_terminated_length": 765.1022186279297, "completions/min_length": 349.25, "completions/min_terminated_length": 349.25, "epoch": 0.5400642222388171, "grad_norm": 18.13139533996582, "kl": 4.07421875, "learning_rate": 9.48753410515424e-06, "loss": 0.1808, "num_tokens": 888196998.0, "reward": 0.6796875298023224, "reward_std": 0.2586808390915394, "rewards/accuracy_reward/mean": 0.20535713993012905, "rewards/accuracy_reward/std": 0.3725249841809273, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4743303656578064, "rewards/tag_count_reward/std": 0.07786679267883301, "step": 1808 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 870.4486999511719, "completions/mean_terminated_length": 775.9113616943359, "completions/min_length": 471.0, "completions/min_terminated_length": 471.0, "epoch": 0.5403629303263385, "grad_norm": 0.4966520369052887, "kl": 2.068359375, "learning_rate": 9.476640437570562e-06, "loss": 0.1045, "num_tokens": 888650911.0, "reward": 0.6372768133878708, "reward_std": 0.12204388901591301, "rewards/accuracy_reward/mean": 0.1450892856810242, "rewards/accuracy_reward/std": 0.2807455938309431, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.041961644776165485, "step": 1809 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 859.7723693847656, "completions/mean_terminated_length": 759.1517028808594, "completions/min_length": 350.5, "completions/min_terminated_length": 350.5, "epoch": 0.5406616384138601, "grad_norm": 0.35822612047195435, "kl": 2.49609375, "learning_rate": 9.465747392738542e-06, "loss": 0.1371, "num_tokens": 889105689.0, "reward": 0.6350446790456772, "reward_std": 0.187143687158823, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.3402218520641327, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 1810 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37276785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 858.4241333007812, "completions/mean_terminated_length": 758.0558166503906, "completions/min_length": 319.75, "completions/min_terminated_length": 319.75, "epoch": 0.5409603465013815, "grad_norm": 0.38839831948280334, "kl": 1.5341796875, "learning_rate": 9.454854983619936e-06, "loss": 0.0795, "num_tokens": 889561479.0, "reward": 0.6891741305589676, "reward_std": 0.12016824074089527, "rewards/accuracy_reward/mean": 0.19866071455180645, "rewards/accuracy_reward/std": 0.38596800714731216, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04537561582401395, "step": 1811 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 861.6339721679688, "completions/mean_terminated_length": 762.3798522949219, "completions/min_length": 336.5, "completions/min_terminated_length": 336.5, "epoch": 0.541259054588903, "grad_norm": 0.2255064845085144, "kl": 2.1953125, "learning_rate": 9.443963223175757e-06, "loss": 0.1108, "num_tokens": 890031251.0, "reward": 0.674107164144516, "reward_std": 0.15986697562038898, "rewards/accuracy_reward/mean": 0.18526785261929035, "rewards/accuracy_reward/std": 0.37626780569553375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.049996999092400074, "step": 1812 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 843.6719207763672, "completions/mean_terminated_length": 738.5049285888672, "completions/min_length": 360.5, "completions/min_terminated_length": 360.5, "epoch": 0.5415577626764244, "grad_norm": 0.6949096918106079, "kl": 1.775390625, "learning_rate": 9.433072124366224e-06, "loss": 0.1078, "num_tokens": 890484480.0, "reward": 0.6573660969734192, "reward_std": 0.1538932491093874, "rewards/accuracy_reward/mean": 0.17187499487772584, "rewards/accuracy_reward/std": 0.3398715369403362, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05665598809719086, "step": 1813 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 883.9353179931641, "completions/mean_terminated_length": 768.1874084472656, "completions/min_length": 284.0, "completions/min_terminated_length": 284.0, "epoch": 0.5418564707639459, "grad_norm": 0.5128798484802246, "kl": 1.923828125, "learning_rate": 9.422181700150798e-06, "loss": 0.1089, "num_tokens": 890963011.0, "reward": 0.6517857611179352, "reward_std": 0.20124569535255432, "rewards/accuracy_reward/mean": 0.1651785708963871, "rewards/accuracy_reward/std": 0.3697928488254547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 1814 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 864.3460235595703, "completions/mean_terminated_length": 743.8656158447266, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "epoch": 0.5421551788514674, "grad_norm": 0.2712235152721405, "kl": 1.892578125, "learning_rate": 9.41129196348811e-06, "loss": 0.1016, "num_tokens": 891418174.0, "reward": 0.6908482611179352, "reward_std": 0.17213033325970173, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.3984857127070427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.0523114912211895, "step": 1815 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 892.2612152099609, "completions/mean_terminated_length": 798.2753448486328, "completions/min_length": 461.5, "completions/min_terminated_length": 461.5, "epoch": 0.5424538869389889, "grad_norm": 0.26969021558761597, "kl": 1.3349609375, "learning_rate": 9.400402927335992e-06, "loss": 0.0603, "num_tokens": 891889155.0, "reward": 0.6484375298023224, "reward_std": 0.1175874974578619, "rewards/accuracy_reward/mean": 0.15625000465661287, "rewards/accuracy_reward/std": 0.25833427533507347, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04008414130657911, "step": 1816 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 885.0045166015625, "completions/mean_terminated_length": 773.1604919433594, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.5427525950265103, "grad_norm": 0.27237269282341003, "kl": 2.435546875, "learning_rate": 9.38951460465143e-06, "loss": 0.1097, "num_tokens": 892353989.0, "reward": 0.701450914144516, "reward_std": 0.17776326835155487, "rewards/accuracy_reward/mean": 0.21428571827709675, "rewards/accuracy_reward/std": 0.38992025703191757, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05382578261196613, "step": 1817 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 873.9353179931641, "completions/mean_terminated_length": 757.7109832763672, "completions/min_length": 383.75, "completions/min_terminated_length": 383.75, "epoch": 0.5430513031140318, "grad_norm": 0.6297151446342468, "kl": 1.80859375, "learning_rate": 9.378627008390575e-06, "loss": 0.086, "num_tokens": 892823208.0, "reward": 0.6333705484867096, "reward_std": 0.12405480933375657, "rewards/accuracy_reward/mean": 0.14285713993012905, "rewards/accuracy_reward/std": 0.277039498090744, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04461357602849603, "step": 1818 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 893.638427734375, "completions/mean_terminated_length": 785.4759368896484, "completions/min_length": 414.0, "completions/min_terminated_length": 414.0, "epoch": 0.5433500112015532, "grad_norm": 0.4188806712627411, "kl": 3.11328125, "learning_rate": 9.367740151508695e-06, "loss": 0.1592, "num_tokens": 893298838.0, "reward": 0.6612723618745804, "reward_std": 0.18845471739768982, "rewards/accuracy_reward/mean": 0.1785714258439839, "rewards/accuracy_reward/std": 0.34151605889201164, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06241532601416111, "step": 1819 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 916.7098541259766, "completions/mean_terminated_length": 806.1717224121094, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "epoch": 0.5436487192890748, "grad_norm": 0.2901591956615448, "kl": 2.09765625, "learning_rate": 9.356854046960194e-06, "loss": 0.1038, "num_tokens": 893783572.0, "reward": 0.618303582072258, "reward_std": 0.1644669696688652, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.33241884410381317, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051574116572737694, "step": 1820 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 891.7857513427734, "completions/mean_terminated_length": 776.4559326171875, "completions/min_length": 457.5, "completions/min_terminated_length": 457.5, "epoch": 0.5439474273765962, "grad_norm": 0.3756887912750244, "kl": 1.4892578125, "learning_rate": 9.34596870769857e-06, "loss": 0.0772, "num_tokens": 894253108.0, "reward": 0.6875000298023224, "reward_std": 0.16943238023668528, "rewards/accuracy_reward/mean": 0.2023809552192688, "rewards/accuracy_reward/std": 0.4014477878808975, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1821 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 900.7545013427734, "completions/mean_terminated_length": 792.2021636962891, "completions/min_length": 415.75, "completions/min_terminated_length": 415.75, "epoch": 0.5442461354641177, "grad_norm": 0.3490296006202698, "kl": 2.244140625, "learning_rate": 9.335084146676422e-06, "loss": 0.1225, "num_tokens": 894734214.0, "reward": 0.5870535969734192, "reward_std": 0.12495180778205395, "rewards/accuracy_reward/mean": 0.10044643003493547, "rewards/accuracy_reward/std": 0.2427675761282444, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148982971907, "step": 1822 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 882.1272735595703, "completions/mean_terminated_length": 760.6163330078125, "completions/min_length": 454.5, "completions/min_terminated_length": 454.5, "epoch": 0.5445448435516391, "grad_norm": 0.2768184244632721, "kl": 2.193359375, "learning_rate": 9.32420037684541e-06, "loss": 0.1088, "num_tokens": 895204527.0, "reward": 0.6250000298023224, "reward_std": 0.1708068773150444, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.339347779750824, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 1823 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 877.0937957763672, "completions/mean_terminated_length": 761.6246948242188, "completions/min_length": 380.25, "completions/min_terminated_length": 380.25, "epoch": 0.5448435516391607, "grad_norm": 0.4428390562534332, "kl": 2.49609375, "learning_rate": 9.313317411156265e-06, "loss": 0.13, "num_tokens": 895678713.0, "reward": 0.6635044813156128, "reward_std": 0.16370990499854088, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.38028907775878906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.052467923145741224, "step": 1824 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 861.8683471679688, "completions/mean_terminated_length": 745.4723968505859, "completions/min_length": 373.75, "completions/min_terminated_length": 373.75, "epoch": 0.5451422597266821, "grad_norm": 0.3387286365032196, "kl": 2.59765625, "learning_rate": 9.302435262558748e-06, "loss": 0.1413, "num_tokens": 896135038.0, "reward": 0.6032366156578064, "reward_std": 0.1716785542666912, "rewards/accuracy_reward/mean": 0.11830356996506453, "rewards/accuracy_reward/std": 0.3156387507915497, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05795421823859215, "step": 1825 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5334821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 913.5491485595703, "completions/mean_terminated_length": 791.4023895263672, "completions/min_length": 422.0, "completions/min_terminated_length": 422.0, "epoch": 0.5454409678142036, "grad_norm": 0.26173263788223267, "kl": 1.890625, "learning_rate": 9.29155394400166e-06, "loss": 0.0841, "num_tokens": 896623956.0, "reward": 0.5747768059372902, "reward_std": 0.14691577479243279, "rewards/accuracy_reward/mean": 0.0848214291036129, "rewards/accuracy_reward/std": 0.2339450977742672, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 1826 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 899.0111999511719, "completions/mean_terminated_length": 742.7496337890625, "completions/min_length": 426.25, "completions/min_terminated_length": 426.25, "epoch": 0.545739675901725, "grad_norm": 0.298147052526474, "kl": 3.34375, "learning_rate": 9.280673468432807e-06, "loss": 0.1714, "num_tokens": 897098041.0, "reward": 0.6316964626312256, "reward_std": 0.17685899510979652, "rewards/accuracy_reward/mean": 0.15401785727590322, "rewards/accuracy_reward/std": 0.33473630249500275, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4776785746216774, "rewards/tag_count_reward/std": 0.07078544236719608, "step": 1827 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 872.9241485595703, "completions/mean_terminated_length": 725.3210144042969, "completions/min_length": 309.75, "completions/min_terminated_length": 309.75, "epoch": 0.5460383839892465, "grad_norm": 0.2588130831718445, "kl": 2.1484375, "learning_rate": 9.269793848798995e-06, "loss": 0.1053, "num_tokens": 897557623.0, "reward": 0.599888414144516, "reward_std": 0.12920022010803223, "rewards/accuracy_reward/mean": 0.11160714272409678, "rewards/accuracy_reward/std": 0.3017156720161438, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.052778348326683044, "step": 1828 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 886.3906555175781, "completions/mean_terminated_length": 768.3572387695312, "completions/min_length": 349.25, "completions/min_terminated_length": 349.25, "epoch": 0.546337092076768, "grad_norm": 0.3039342761039734, "kl": 1.955078125, "learning_rate": 9.258915098046008e-06, "loss": 0.1079, "num_tokens": 898024198.0, "reward": 0.6266741305589676, "reward_std": 0.1411457657814026, "rewards/accuracy_reward/mean": 0.13616071362048388, "rewards/accuracy_reward/std": 0.329923577606678, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 1829 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 879.5178833007812, "completions/mean_terminated_length": 752.0581970214844, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "epoch": 0.5466358001642895, "grad_norm": 0.2525142431259155, "kl": 1.7265625, "learning_rate": 9.248037229118602e-06, "loss": 0.0958, "num_tokens": 898492046.0, "reward": 0.5284598469734192, "reward_std": 0.10505878366529942, "rewards/accuracy_reward/mean": 0.03720238176174462, "rewards/accuracy_reward/std": 0.18224875815212727, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.0408577430061996, "step": 1830 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 895.5937957763672, "completions/mean_terminated_length": 773.1215972900391, "completions/min_length": 405.5, "completions/min_terminated_length": 405.5, "epoch": 0.5469345082518109, "grad_norm": 0.3366168439388275, "kl": 2.64453125, "learning_rate": 9.237160254960477e-06, "loss": 0.1374, "num_tokens": 898961272.0, "reward": 0.6434152126312256, "reward_std": 0.21515673398971558, "rewards/accuracy_reward/mean": 0.16108631435781717, "rewards/accuracy_reward/std": 0.34405651688575745, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 1831 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 876.747802734375, "completions/mean_terminated_length": 720.3404388427734, "completions/min_length": 333.75, "completions/min_terminated_length": 333.75, "epoch": 0.5472332163393324, "grad_norm": 0.2797817885875702, "kl": 2.25390625, "learning_rate": 9.226284188514277e-06, "loss": 0.1161, "num_tokens": 899430087.0, "reward": 0.6640625298023224, "reward_std": 0.13289980217814445, "rewards/accuracy_reward/mean": 0.17410714644938707, "rewards/accuracy_reward/std": 0.35423441231250763, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04923219420015812, "step": 1832 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 857.2076263427734, "completions/mean_terminated_length": 732.3214416503906, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "epoch": 0.5475319244268538, "grad_norm": 0.24840015172958374, "kl": 1.98876953125, "learning_rate": 9.215409042721553e-06, "loss": 0.0987, "num_tokens": 899891092.0, "reward": 0.6718750298023224, "reward_std": 0.14695269241929054, "rewards/accuracy_reward/mean": 0.18303571385331452, "rewards/accuracy_reward/std": 0.32348055206239223, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.044330806471407413, "step": 1833 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 869.5536193847656, "completions/mean_terminated_length": 761.9212341308594, "completions/min_length": 361.25, "completions/min_terminated_length": 361.25, "epoch": 0.5478306325143754, "grad_norm": 0.2894299030303955, "kl": 2.009765625, "learning_rate": 9.204534830522772e-06, "loss": 0.1073, "num_tokens": 900352284.0, "reward": 0.6311384290456772, "reward_std": 0.2030741199851036, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.3337591215968132, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050074178259819746, "step": 1834 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 861.3727874755859, "completions/mean_terminated_length": 757.5978393554688, "completions/min_length": 339.25, "completions/min_terminated_length": 339.25, "epoch": 0.5481293406018968, "grad_norm": 0.3562660217285156, "kl": 1.5859375, "learning_rate": 9.193661564857283e-06, "loss": 0.1016, "num_tokens": 900807043.0, "reward": 0.690848246216774, "reward_std": 0.143521498888731, "rewards/accuracy_reward/mean": 0.19866071362048388, "rewards/accuracy_reward/std": 0.3623369187116623, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 1835 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3258928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 844.4241485595703, "completions/mean_terminated_length": 757.9283294677734, "completions/min_length": 351.75, "completions/min_terminated_length": 351.75, "epoch": 0.5484280486894183, "grad_norm": 0.3914182186126709, "kl": 1.908203125, "learning_rate": 9.182789258663321e-06, "loss": 0.1064, "num_tokens": 901248737.0, "reward": 0.635044664144516, "reward_std": 0.13470162823796272, "rewards/accuracy_reward/mean": 0.14508928591385484, "rewards/accuracy_reward/std": 0.3131481818854809, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 1836 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 825.2790374755859, "completions/mean_terminated_length": 733.6519622802734, "completions/min_length": 388.5, "completions/min_terminated_length": 388.5, "epoch": 0.5487267567769397, "grad_norm": 0.21167419850826263, "kl": 2.259765625, "learning_rate": 9.17191792487796e-06, "loss": 0.1288, "num_tokens": 901684302.0, "reward": 0.647879496216774, "reward_std": 0.1818141955882311, "rewards/accuracy_reward/mean": 0.15848213899880648, "rewards/accuracy_reward/std": 0.34250105917453766, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932655245066, "step": 1837 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 870.263427734375, "completions/mean_terminated_length": 781.8807525634766, "completions/min_length": 381.25, "completions/min_terminated_length": 381.25, "epoch": 0.5490254648644612, "grad_norm": 0.2661707401275635, "kl": 2.234375, "learning_rate": 9.16104757643713e-06, "loss": 0.1195, "num_tokens": 902150468.0, "reward": 0.6523437947034836, "reward_std": 0.16847142204642296, "rewards/accuracy_reward/mean": 0.16294642630964518, "rewards/accuracy_reward/std": 0.32328175753355026, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810389876366, "step": 1838 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 851.7522735595703, "completions/mean_terminated_length": 770.0933685302734, "completions/min_length": 336.25, "completions/min_terminated_length": 336.25, "epoch": 0.5493241729519827, "grad_norm": 0.3207106590270996, "kl": 1.994140625, "learning_rate": 9.150178226275584e-06, "loss": 0.1004, "num_tokens": 902600453.0, "reward": 0.6277901977300644, "reward_std": 0.13208172749727964, "rewards/accuracy_reward/mean": 0.13616071548312902, "rewards/accuracy_reward/std": 0.3336229771375656, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 1839 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 820.4732513427734, "completions/mean_terminated_length": 731.8497009277344, "completions/min_length": 367.25, "completions/min_terminated_length": 367.25, "epoch": 0.5496228810395042, "grad_norm": 0.266274631023407, "kl": 2.03125, "learning_rate": 9.139309887326894e-06, "loss": 0.1064, "num_tokens": 903036905.0, "reward": 0.6300223618745804, "reward_std": 0.15221986174583435, "rewards/accuracy_reward/mean": 0.1383928540162742, "rewards/accuracy_reward/std": 0.3176145479083061, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 1840 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 845.6808471679688, "completions/mean_terminated_length": 767.5760192871094, "completions/min_length": 389.75, "completions/min_terminated_length": 389.75, "epoch": 0.5499215891270256, "grad_norm": 0.29090768098831177, "kl": 2.30078125, "learning_rate": 9.128442572523418e-06, "loss": 0.1224, "num_tokens": 903487290.0, "reward": 0.5954241454601288, "reward_std": 0.14574345760047436, "rewards/accuracy_reward/mean": 0.10714286006987095, "rewards/accuracy_reward/std": 0.3072098195552826, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 1841 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.25223214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 801.0870819091797, "completions/mean_terminated_length": 729.7790222167969, "completions/min_length": 271.0, "completions/min_terminated_length": 271.0, "epoch": 0.5502202972145471, "grad_norm": 0.49876540899276733, "kl": 3.25, "learning_rate": 9.117576294796307e-06, "loss": 0.1759, "num_tokens": 903916849.0, "reward": 0.680245578289032, "reward_std": 0.18279768526554108, "rewards/accuracy_reward/mean": 0.1941964328289032, "rewards/accuracy_reward/std": 0.379142876714468, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05749546363949776, "step": 1842 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.26785714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 830.7143249511719, "completions/mean_terminated_length": 763.8059387207031, "completions/min_length": 400.25, "completions/min_terminated_length": 400.25, "epoch": 0.5505190053020685, "grad_norm": 0.2890293002128601, "kl": 2.4140625, "learning_rate": 9.106711067075464e-06, "loss": 0.1356, "num_tokens": 904358001.0, "reward": 0.6177455633878708, "reward_std": 0.14142770040780306, "rewards/accuracy_reward/mean": 0.1272321455180645, "rewards/accuracy_reward/std": 0.32999154180288315, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1843 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 833.3995819091797, "completions/mean_terminated_length": 751.8958435058594, "completions/min_length": 398.75, "completions/min_terminated_length": 398.75, "epoch": 0.5508177133895901, "grad_norm": 0.18619225919246674, "kl": 1.99609375, "learning_rate": 9.095846902289556e-06, "loss": 0.1142, "num_tokens": 904803636.0, "reward": 0.6808036118745804, "reward_std": 0.22581946104764938, "rewards/accuracy_reward/mean": 0.18973213899880648, "rewards/accuracy_reward/std": 0.3675852492451668, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.046403173357248306, "step": 1844 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2723214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 812.888427734375, "completions/mean_terminated_length": 731.9304046630859, "completions/min_length": 354.75, "completions/min_terminated_length": 354.75, "epoch": 0.5511164214771115, "grad_norm": 0.3673773407936096, "kl": 2.6796875, "learning_rate": 9.084983813365977e-06, "loss": 0.1414, "num_tokens": 905233938.0, "reward": 0.6618303805589676, "reward_std": 0.18778668157756329, "rewards/accuracy_reward/mean": 0.17187500279396772, "rewards/accuracy_reward/std": 0.3585606664419174, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886399567127, "step": 1845 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 875.6607360839844, "completions/mean_terminated_length": 788.1026153564453, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "epoch": 0.551415129564633, "grad_norm": 0.22735682129859924, "kl": 2.69921875, "learning_rate": 9.074121813230846e-06, "loss": 0.1325, "num_tokens": 905700010.0, "reward": 0.6194196790456772, "reward_std": 0.1247143903747201, "rewards/accuracy_reward/mean": 0.14285714412108064, "rewards/accuracy_reward/std": 0.25715720281004906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.054059810005128384, "step": 1846 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 879.8683471679688, "completions/mean_terminated_length": 774.1047058105469, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "epoch": 0.5517138376521544, "grad_norm": 0.30954158306121826, "kl": 2.080078125, "learning_rate": 9.06326091480898e-06, "loss": 0.117, "num_tokens": 906165135.0, "reward": 0.6210937798023224, "reward_std": 0.0987859689630568, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.2844284325838089, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04868226684629917, "step": 1847 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 869.5826263427734, "completions/mean_terminated_length": 788.2125091552734, "completions/min_length": 422.25, "completions/min_terminated_length": 422.25, "epoch": 0.552012545739676, "grad_norm": 0.2288322001695633, "kl": 1.9228515625, "learning_rate": 9.05240113102389e-06, "loss": 0.1015, "num_tokens": 906629220.0, "reward": 0.5876116454601288, "reward_std": 0.12623553350567818, "rewards/accuracy_reward/mean": 0.09821428451687098, "rewards/accuracy_reward/std": 0.2851461060345173, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04831482237204909, "step": 1848 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 862.6942443847656, "completions/mean_terminated_length": 779.8329772949219, "completions/min_length": 473.0, "completions/min_terminated_length": 473.0, "epoch": 0.5523112538271974, "grad_norm": 0.2219460904598236, "kl": 2.15625, "learning_rate": 9.04154247479776e-06, "loss": 0.1082, "num_tokens": 907090731.0, "reward": 0.6411830633878708, "reward_std": 0.10862840712070465, "rewards/accuracy_reward/mean": 0.1517857159487903, "rewards/accuracy_reward/std": 0.3340533971786499, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04690983286127448, "step": 1849 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 863.4911041259766, "completions/mean_terminated_length": 780.0639801025391, "completions/min_length": 443.5, "completions/min_terminated_length": 443.5, "epoch": 0.5526099619147189, "grad_norm": 0.32670772075653076, "kl": 2.13671875, "learning_rate": 9.030684959051438e-06, "loss": 0.1139, "num_tokens": 907547719.0, "reward": 0.6668527126312256, "reward_std": 0.19173459336161613, "rewards/accuracy_reward/mean": 0.176339291036129, "rewards/accuracy_reward/std": 0.37472253292798996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047120303846895695, "step": 1850 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 865.5022735595703, "completions/mean_terminated_length": 760.0141448974609, "completions/min_length": 399.5, "completions/min_terminated_length": 399.5, "epoch": 0.5529086700022403, "grad_norm": 0.21041610836982727, "kl": 1.71484375, "learning_rate": 9.019828596704394e-06, "loss": 0.09, "num_tokens": 908004328.0, "reward": 0.6155134290456772, "reward_std": 0.1513044685125351, "rewards/accuracy_reward/mean": 0.12276785750873387, "rewards/accuracy_reward/std": 0.2972430866211653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 1851 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 875.9866333007812, "completions/mean_terminated_length": 773.1859130859375, "completions/min_length": 412.0, "completions/min_terminated_length": 412.0, "epoch": 0.5532073780897617, "grad_norm": 0.25734591484069824, "kl": 2.2890625, "learning_rate": 9.008973400674752e-06, "loss": 0.1323, "num_tokens": 908461906.0, "reward": 0.6210937798023224, "reward_std": 0.16722634993493557, "rewards/accuracy_reward/mean": 0.1339285741560161, "rewards/accuracy_reward/std": 0.32121996581554413, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.054169113747775555, "step": 1852 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 905.6272735595703, "completions/mean_terminated_length": 795.7050170898438, "completions/min_length": 437.5, "completions/min_terminated_length": 437.5, "epoch": 0.5535060861772833, "grad_norm": 0.46676090359687805, "kl": 1.3173828125, "learning_rate": 8.99811938387924e-06, "loss": 0.0761, "num_tokens": 908943083.0, "reward": 0.6049107536673546, "reward_std": 0.09964764304459095, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.267510250210762, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.493303582072258, "rewards/tag_count_reward/std": 0.03774221893399954, "step": 1853 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 853.5848541259766, "completions/mean_terminated_length": 747.3095245361328, "completions/min_length": 334.25, "completions/min_terminated_length": 334.25, "epoch": 0.5538047942648047, "grad_norm": 0.40031367540359497, "kl": 1.67578125, "learning_rate": 8.987266559233166e-06, "loss": 0.094, "num_tokens": 909400161.0, "reward": 0.7187500298023224, "reward_std": 0.1564230341464281, "rewards/accuracy_reward/mean": 0.2332589253783226, "rewards/accuracy_reward/std": 0.41002538800239563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04660273250192404, "step": 1854 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 870.8750152587891, "completions/mean_terminated_length": 779.2831268310547, "completions/min_length": 427.25, "completions/min_terminated_length": 427.25, "epoch": 0.5541035023523262, "grad_norm": 0.22354450821876526, "kl": 1.509765625, "learning_rate": 8.976414939650443e-06, "loss": 0.0699, "num_tokens": 909857673.0, "reward": 0.6222098469734192, "reward_std": 0.15941080637276173, "rewards/accuracy_reward/mean": 0.12946428637951612, "rewards/accuracy_reward/std": 0.3058853894472122, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 1855 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45535714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 892.9308319091797, "completions/mean_terminated_length": 790.6443786621094, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "epoch": 0.5544022104398476, "grad_norm": 0.4693440794944763, "kl": 1.822265625, "learning_rate": 8.965564538043535e-06, "loss": 0.1007, "num_tokens": 910327146.0, "reward": 0.5524553805589676, "reward_std": 0.10101372841745615, "rewards/accuracy_reward/mean": 0.06026785774156451, "rewards/accuracy_reward/std": 0.19186493940651417, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043574148789048195, "step": 1856 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 844.7835235595703, "completions/mean_terminated_length": 733.4343414306641, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "epoch": 0.5547009185273691, "grad_norm": 0.24252785742282867, "kl": 2.0078125, "learning_rate": 8.954715367323468e-06, "loss": 0.1081, "num_tokens": 910772809.0, "reward": 0.7656250298023224, "reward_std": 0.15703747048974037, "rewards/accuracy_reward/mean": 0.2745535634458065, "rewards/accuracy_reward/std": 0.4458833113312721, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 1857 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 858.9687805175781, "completions/mean_terminated_length": 756.0959777832031, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.5549996266148906, "grad_norm": 0.26393595337867737, "kl": 1.943359375, "learning_rate": 8.943867440399787e-06, "loss": 0.1036, "num_tokens": 911227963.0, "reward": 0.713169664144516, "reward_std": 0.24607159569859505, "rewards/accuracy_reward/mean": 0.22916666977107525, "rewards/accuracy_reward/std": 0.39308469742536545, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 1858 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 882.1629791259766, "completions/mean_terminated_length": 771.0470123291016, "completions/min_length": 343.5, "completions/min_terminated_length": 343.5, "epoch": 0.5552983347024121, "grad_norm": 0.3017750084400177, "kl": 2.82421875, "learning_rate": 8.933020770180574e-06, "loss": 0.132, "num_tokens": 911692788.0, "reward": 0.7042410969734192, "reward_std": 0.11381690204143524, "rewards/accuracy_reward/mean": 0.21651786006987095, "rewards/accuracy_reward/std": 0.38613060116767883, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 1859 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 834.3861999511719, "completions/mean_terminated_length": 731.8650665283203, "completions/min_length": 302.25, "completions/min_terminated_length": 302.25, "epoch": 0.5555970427899335, "grad_norm": 0.31974732875823975, "kl": 2.2109375, "learning_rate": 8.922175369572407e-06, "loss": 0.1168, "num_tokens": 912150097.0, "reward": 0.627232164144516, "reward_std": 0.0982205793261528, "rewards/accuracy_reward/mean": 0.13616071455180645, "rewards/accuracy_reward/std": 0.2864500880241394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.03677344275638461, "step": 1860 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 839.6339569091797, "completions/mean_terminated_length": 724.9265594482422, "completions/min_length": 281.75, "completions/min_terminated_length": 281.75, "epoch": 0.555895750877455, "grad_norm": 0.27676305174827576, "kl": 2.912109375, "learning_rate": 8.911331251480357e-06, "loss": 0.1458, "num_tokens": 912602717.0, "reward": 0.6718750447034836, "reward_std": 0.20133788138628006, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.38333045691251755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05543891713023186, "step": 1861 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 880.0870819091797, "completions/mean_terminated_length": 769.5385131835938, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "epoch": 0.5561944589649764, "grad_norm": 0.295040488243103, "kl": 1.734375, "learning_rate": 8.90048842880796e-06, "loss": 0.0828, "num_tokens": 913066964.0, "reward": 0.699776828289032, "reward_std": 0.17991334851831198, "rewards/accuracy_reward/mean": 0.21912202355451882, "rewards/accuracy_reward/std": 0.36230177991092205, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 1862 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 843.5446929931641, "completions/mean_terminated_length": 760.9228057861328, "completions/min_length": 403.25, "completions/min_terminated_length": 403.25, "epoch": 0.556493167052498, "grad_norm": 0.2728697657585144, "kl": 1.677734375, "learning_rate": 8.889646914457225e-06, "loss": 0.092, "num_tokens": 913510840.0, "reward": 0.6400670111179352, "reward_std": 0.14457618817687035, "rewards/accuracy_reward/mean": 0.14732143050059676, "rewards/accuracy_reward/std": 0.3092727102339268, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 1863 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 912.9308471679688, "completions/mean_terminated_length": 821.1961975097656, "completions/min_length": 444.0, "completions/min_terminated_length": 444.0, "epoch": 0.5567918751400194, "grad_norm": 0.17460933327674866, "kl": 1.63671875, "learning_rate": 8.87880672132859e-06, "loss": 0.0789, "num_tokens": 914004233.0, "reward": 0.6551339626312256, "reward_std": 0.14013028144836426, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.30571267753839493, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.04197291610762477, "step": 1864 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 867.7500305175781, "completions/mean_terminated_length": 773.3712310791016, "completions/min_length": 326.75, "completions/min_terminated_length": 326.75, "epoch": 0.5570905832275409, "grad_norm": 0.331245094537735, "kl": 1.49462890625, "learning_rate": 8.867967862320935e-06, "loss": 0.0815, "num_tokens": 914463833.0, "reward": 0.5485491305589676, "reward_std": 0.10064542014151812, "rewards/accuracy_reward/mean": 0.05580357275903225, "rewards/accuracy_reward/std": 0.18331189453601837, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03626677952706814, "step": 1865 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 867.2321929931641, "completions/mean_terminated_length": 780.7046051025391, "completions/min_length": 477.75, "completions/min_terminated_length": 477.75, "epoch": 0.5573892913150623, "grad_norm": 0.1946997493505478, "kl": 0.9169921875, "learning_rate": 8.857130350331535e-06, "loss": 0.056, "num_tokens": 914927569.0, "reward": 0.7064732313156128, "reward_std": 0.1650975216180086, "rewards/accuracy_reward/mean": 0.20982142724096775, "rewards/accuracy_reward/std": 0.39366383105516434, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.496651791036129, "rewards/tag_count_reward/std": 0.027853476349264383, "step": 1866 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33035714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 829.1295013427734, "completions/mean_terminated_length": 738.4530334472656, "completions/min_length": 301.5, "completions/min_terminated_length": 301.5, "epoch": 0.5576879994025838, "grad_norm": 0.2288326770067215, "kl": 2.00390625, "learning_rate": 8.846294198256085e-06, "loss": 0.0955, "num_tokens": 915370379.0, "reward": 0.7477678805589676, "reward_std": 0.19721683114767075, "rewards/accuracy_reward/mean": 0.2589285746216774, "rewards/accuracy_reward/std": 0.41156020015478134, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05146361608058214, "step": 1867 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 850.9531555175781, "completions/mean_terminated_length": 758.5759124755859, "completions/min_length": 383.25, "completions/min_terminated_length": 383.25, "epoch": 0.5579867074901053, "grad_norm": 0.47014161944389343, "kl": 1.310546875, "learning_rate": 8.835459418988635e-06, "loss": 0.0722, "num_tokens": 915821702.0, "reward": 0.6272321790456772, "reward_std": 0.11065852176398039, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.264439158141613, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1868 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3258928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 851.0469055175781, "completions/mean_terminated_length": 769.2011566162109, "completions/min_length": 362.5, "completions/min_terminated_length": 362.5, "epoch": 0.5582854155776268, "grad_norm": 0.20663312077522278, "kl": 1.2939453125, "learning_rate": 8.824626025421625e-06, "loss": 0.0643, "num_tokens": 916279659.0, "reward": 0.6099330633878708, "reward_std": 0.12544633192010224, "rewards/accuracy_reward/mean": 0.11607143003493547, "rewards/accuracy_reward/std": 0.2514657825231552, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1869 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 882.9933471679688, "completions/mean_terminated_length": 787.7966156005859, "completions/min_length": 460.5, "completions/min_terminated_length": 460.5, "epoch": 0.5585841236651482, "grad_norm": 0.2854183316230774, "kl": 1.248046875, "learning_rate": 8.81379403044583e-06, "loss": 0.065, "num_tokens": 916754232.0, "reward": 0.601004496216774, "reward_std": 0.08375841751694679, "rewards/accuracy_reward/mean": 0.10491071292199194, "rewards/accuracy_reward/std": 0.27026291750371456, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1870 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 903.9799499511719, "completions/mean_terminated_length": 803.9713439941406, "completions/min_length": 485.75, "completions/min_terminated_length": 485.75, "epoch": 0.5588828317526697, "grad_norm": 0.3506425619125366, "kl": 0.86181640625, "learning_rate": 8.802963446950378e-06, "loss": 0.0406, "num_tokens": 917235791.0, "reward": 0.6445312798023224, "reward_std": 0.14159296080470085, "rewards/accuracy_reward/mean": 0.14732142817229033, "rewards/accuracy_reward/std": 0.33450905978679657, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 1871 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 882.9397735595703, "completions/mean_terminated_length": 784.6172180175781, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.5591815398401911, "grad_norm": 0.206809401512146, "kl": 0.9443359375, "learning_rate": 8.792134287822693e-06, "loss": 0.0524, "num_tokens": 917708500.0, "reward": 0.555803582072258, "reward_std": 0.08748714113608003, "rewards/accuracy_reward/mean": 0.06101190438494086, "rewards/accuracy_reward/std": 0.19368072971701622, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357015132904, "rewards/tag_count_reward/std": 0.033256832510232925, "step": 1872 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 860.0692291259766, "completions/mean_terminated_length": 765.2215881347656, "completions/min_length": 389.5, "completions/min_terminated_length": 389.5, "epoch": 0.5594802479277127, "grad_norm": 0.39325767755508423, "kl": 1.5849609375, "learning_rate": 8.781306565948528e-06, "loss": 0.0788, "num_tokens": 918159859.0, "reward": 0.6495535969734192, "reward_std": 0.16421638615429401, "rewards/accuracy_reward/mean": 0.15625000186264515, "rewards/accuracy_reward/std": 0.3471071124076843, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1873 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37276785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 864.5826263427734, "completions/mean_terminated_length": 779.9187164306641, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "epoch": 0.5597789560152341, "grad_norm": 0.20550711452960968, "kl": 1.1708984375, "learning_rate": 8.770480294211909e-06, "loss": 0.0612, "num_tokens": 918613160.0, "reward": 0.6277902275323868, "reward_std": 0.16402830183506012, "rewards/accuracy_reward/mean": 0.14434523694217205, "rewards/accuracy_reward/std": 0.3351188004016876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.033437756821513176, "step": 1874 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 879.419677734375, "completions/mean_terminated_length": 779.6178741455078, "completions/min_length": 394.25, "completions/min_terminated_length": 394.25, "epoch": 0.5600776641027556, "grad_norm": 0.19666461646556854, "kl": 1.447265625, "learning_rate": 8.75965548549515e-06, "loss": 0.076, "num_tokens": 919082468.0, "reward": 0.7148437723517418, "reward_std": 0.2053431337699294, "rewards/accuracy_reward/mean": 0.2232142798602581, "rewards/accuracy_reward/std": 0.32867879420518875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 1875 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 893.0781707763672, "completions/mean_terminated_length": 792.3890075683594, "completions/min_length": 375.5, "completions/min_terminated_length": 375.5, "epoch": 0.560376372190277, "grad_norm": 0.19666758179664612, "kl": 1.3603515625, "learning_rate": 8.74883215267881e-06, "loss": 0.0627, "num_tokens": 919555927.0, "reward": 0.6183036118745804, "reward_std": 0.1544569544494152, "rewards/accuracy_reward/mean": 0.12499999813735485, "rewards/accuracy_reward/std": 0.26713576167821884, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1876 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 887.3348541259766, "completions/mean_terminated_length": 794.9699249267578, "completions/min_length": 372.5, "completions/min_terminated_length": 372.5, "epoch": 0.5606750802777986, "grad_norm": 0.19746172428131104, "kl": 0.85595703125, "learning_rate": 8.738010308641705e-06, "loss": 0.0536, "num_tokens": 920022701.0, "reward": 0.6400669813156128, "reward_std": 0.0824909801594913, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.3487042188644409, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 1877 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 896.1674499511719, "completions/mean_terminated_length": 781.296875, "completions/min_length": 400.75, "completions/min_terminated_length": 400.75, "epoch": 0.56097378836532, "grad_norm": 0.5079187154769897, "kl": 1.865234375, "learning_rate": 8.72718996626087e-06, "loss": 0.1034, "num_tokens": 920505384.0, "reward": 0.6250000223517418, "reward_std": 0.15939313266426325, "rewards/accuracy_reward/mean": 0.13616071548312902, "rewards/accuracy_reward/std": 0.272699199616909, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.049996999092400074, "step": 1878 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 904.7232666015625, "completions/mean_terminated_length": 801.1186828613281, "completions/min_length": 351.75, "completions/min_terminated_length": 351.75, "epoch": 0.5612724964528415, "grad_norm": 0.20887361466884613, "kl": 1.7265625, "learning_rate": 8.716371138411557e-06, "loss": 0.0793, "num_tokens": 920985356.0, "reward": 0.659598246216774, "reward_std": 0.16904843971133232, "rewards/accuracy_reward/mean": 0.1696428544819355, "rewards/accuracy_reward/std": 0.3562445342540741, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 1879 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 908.5111999511719, "completions/mean_terminated_length": 822.8502349853516, "completions/min_length": 417.0, "completions/min_terminated_length": 417.0, "epoch": 0.5615712045403629, "grad_norm": 0.23169274628162384, "kl": 1.90625, "learning_rate": 8.705553837967218e-06, "loss": 0.097, "num_tokens": 921471633.0, "reward": 0.7511161044239998, "reward_std": 0.21070499811321497, "rewards/accuracy_reward/mean": 0.2633928507566452, "rewards/accuracy_reward/std": 0.3450101688504219, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 1880 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 888.5647583007812, "completions/mean_terminated_length": 774.8798065185547, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.5618699126278844, "grad_norm": 0.17600686848163605, "kl": 1.6376953125, "learning_rate": 8.694738077799487e-06, "loss": 0.0795, "num_tokens": 921935870.0, "reward": 0.6378348469734192, "reward_std": 0.1721804291009903, "rewards/accuracy_reward/mean": 0.1473214328289032, "rewards/accuracy_reward/std": 0.3279986009001732, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04537529917433858, "step": 1881 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.59375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 955.7366485595703, "completions/mean_terminated_length": 855.5046997070312, "completions/min_length": 548.75, "completions/min_terminated_length": 548.75, "epoch": 0.5621686207154059, "grad_norm": 0.32964885234832764, "kl": 2.5625, "learning_rate": 8.683923870778155e-06, "loss": 0.1153, "num_tokens": 922429272.0, "reward": 0.6060268133878708, "reward_std": 0.1683004079386592, "rewards/accuracy_reward/mean": 0.12499999813735485, "rewards/accuracy_reward/std": 0.26958145946264267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4810267835855484, "rewards/tag_count_reward/std": 0.06493131443858147, "step": 1882 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 936.7388916015625, "completions/mean_terminated_length": 825.7047424316406, "completions/min_length": 556.5, "completions/min_terminated_length": 556.5, "epoch": 0.5624673288029274, "grad_norm": 0.3583040237426758, "kl": 2.21875, "learning_rate": 8.673111229771182e-06, "loss": 0.1198, "num_tokens": 922927891.0, "reward": 0.6250000298023224, "reward_std": 0.15197227522730827, "rewards/accuracy_reward/mean": 0.14062499743886292, "rewards/accuracy_reward/std": 0.3101118244230747, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.062069219537079334, "step": 1883 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5848214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 926.4955749511719, "completions/mean_terminated_length": 789.4566955566406, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "epoch": 0.5627660368904488, "grad_norm": 0.4966598451137543, "kl": 2.96875, "learning_rate": 8.66230016764465e-06, "loss": 0.1579, "num_tokens": 923410721.0, "reward": 0.6545759290456772, "reward_std": 0.19067857041954994, "rewards/accuracy_reward/mean": 0.17410714365541935, "rewards/accuracy_reward/std": 0.36712101101875305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4804687425494194, "rewards/tag_count_reward/std": 0.06723255570977926, "step": 1884 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 931.2120819091797, "completions/mean_terminated_length": 821.7663269042969, "completions/min_length": 454.5, "completions/min_terminated_length": 454.5, "epoch": 0.5630647449779703, "grad_norm": 0.19632147252559662, "kl": 2.3232421875, "learning_rate": 8.651490697262773e-06, "loss": 0.112, "num_tokens": 923894064.0, "reward": 0.6657366454601288, "reward_std": 0.18352700397372246, "rewards/accuracy_reward/mean": 0.17857142630964518, "rewards/accuracy_reward/std": 0.34363533556461334, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05230560014024377, "step": 1885 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 925.6585083007812, "completions/mean_terminated_length": 801.0438537597656, "completions/min_length": 470.5, "completions/min_terminated_length": 470.5, "epoch": 0.5633634530654917, "grad_norm": 0.8444082140922546, "kl": 4.0390625, "learning_rate": 8.64068283148786e-06, "loss": 0.1798, "num_tokens": 924377543.0, "reward": 0.5848214477300644, "reward_std": 0.14666638989001513, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.2653872147202492, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4754464328289032, "rewards/tag_count_reward/std": 0.07247901055961847, "step": 1886 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 889.3616638183594, "completions/mean_terminated_length": 756.7527618408203, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.5636621611530133, "grad_norm": 0.31482312083244324, "kl": 3.166015625, "learning_rate": 8.629876583180322e-06, "loss": 0.164, "num_tokens": 924845129.0, "reward": 0.6958705633878708, "reward_std": 0.19902322068810463, "rewards/accuracy_reward/mean": 0.21205357275903225, "rewards/accuracy_reward/std": 0.3904358521103859, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06139749940484762, "step": 1887 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 929.2991485595703, "completions/mean_terminated_length": 825.6412811279297, "completions/min_length": 495.5, "completions/min_terminated_length": 495.5, "epoch": 0.5639608692405347, "grad_norm": 0.7372589707374573, "kl": 3.83203125, "learning_rate": 8.619071965198635e-06, "loss": 0.1651, "num_tokens": 925336671.0, "reward": 0.6143973395228386, "reward_std": 0.15326854586601257, "rewards/accuracy_reward/mean": 0.13392857648432255, "rewards/accuracy_reward/std": 0.2789922505617142, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4804687574505806, "rewards/tag_count_reward/std": 0.06606535613536835, "step": 1888 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49553571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 906.0915679931641, "completions/mean_terminated_length": 793.5121917724609, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.5642595773280562, "grad_norm": 0.2713833451271057, "kl": 3.859375, "learning_rate": 8.60826899039935e-06, "loss": 0.1966, "num_tokens": 925813672.0, "reward": 0.5887277200818062, "reward_std": 0.1737117227166891, "rewards/accuracy_reward/mean": 0.11160714109428227, "rewards/accuracy_reward/std": 0.281693946570158, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4771205335855484, "rewards/tag_count_reward/std": 0.07161347568035126, "step": 1889 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 890.6317291259766, "completions/mean_terminated_length": 785.1650085449219, "completions/min_length": 444.25, "completions/min_terminated_length": 444.25, "epoch": 0.5645582854155776, "grad_norm": 0.2826297879219055, "kl": 3.69140625, "learning_rate": 8.597467671637037e-06, "loss": 0.1901, "num_tokens": 926282883.0, "reward": 0.6300223469734192, "reward_std": 0.1739109493792057, "rewards/accuracy_reward/mean": 0.1588541679084301, "rewards/accuracy_reward/std": 0.35385801270604134, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366007566452, "rewards/tag_count_reward/std": 0.07009305339306593, "step": 1890 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45535714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 886.6027221679688, "completions/mean_terminated_length": 777.3822479248047, "completions/min_length": 347.75, "completions/min_terminated_length": 347.75, "epoch": 0.5648569935030991, "grad_norm": 0.29992392659187317, "kl": 2.2470703125, "learning_rate": 8.586668021764328e-06, "loss": 0.109, "num_tokens": 926754369.0, "reward": 0.6573660969734192, "reward_std": 0.1869601011276245, "rewards/accuracy_reward/mean": 0.16964285727590322, "rewards/accuracy_reward/std": 0.3475473001599312, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05168584827333689, "step": 1891 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 904.2567291259766, "completions/mean_terminated_length": 798.5682983398438, "completions/min_length": 452.0, "completions/min_terminated_length": 452.0, "epoch": 0.5651557015906206, "grad_norm": 0.5677658319473267, "kl": 3.44140625, "learning_rate": 8.575870053631841e-06, "loss": 0.168, "num_tokens": 927231700.0, "reward": 0.6595982313156128, "reward_std": 0.1844778023660183, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.3850152939558029, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.06869727186858654, "step": 1892 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 861.9174499511719, "completions/mean_terminated_length": 750.4871673583984, "completions/min_length": 391.5, "completions/min_terminated_length": 391.5, "epoch": 0.5654544096781421, "grad_norm": 0.3904981315135956, "kl": 1.849609375, "learning_rate": 8.56507378008821e-06, "loss": 0.0986, "num_tokens": 927683919.0, "reward": 0.636160746216774, "reward_std": 0.10545609425753355, "rewards/accuracy_reward/mean": 0.1495535741560161, "rewards/accuracy_reward/std": 0.3338511511683464, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05548384319990873, "step": 1893 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 886.5290374755859, "completions/mean_terminated_length": 772.5963745117188, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "epoch": 0.5657531177656635, "grad_norm": 0.328525573015213, "kl": 2.1875, "learning_rate": 8.554279213980041e-06, "loss": 0.1137, "num_tokens": 928153404.0, "reward": 0.672433078289032, "reward_std": 0.199325542896986, "rewards/accuracy_reward/mean": 0.2012648768723011, "rewards/accuracy_reward/std": 0.3993125334382057, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.059022306464612484, "step": 1894 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 846.9643249511719, "completions/mean_terminated_length": 750.483642578125, "completions/min_length": 363.25, "completions/min_terminated_length": 363.25, "epoch": 0.5660518258531849, "grad_norm": 0.5282795429229736, "kl": 2.1015625, "learning_rate": 8.543486368151926e-06, "loss": 0.1219, "num_tokens": 928607564.0, "reward": 0.6478794887661934, "reward_std": 0.1547401025891304, "rewards/accuracy_reward/mean": 0.1629464328289032, "rewards/accuracy_reward/std": 0.3103029429912567, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05936532001942396, "step": 1895 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 900.6562957763672, "completions/mean_terminated_length": 806.65869140625, "completions/min_length": 457.5, "completions/min_terminated_length": 457.5, "epoch": 0.5663505339407064, "grad_norm": 0.2803820073604584, "kl": 2.322265625, "learning_rate": 8.532695255446384e-06, "loss": 0.1175, "num_tokens": 929088194.0, "reward": 0.5630580633878708, "reward_std": 0.12128463201224804, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.2664731852710247, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.0580708310008049, "step": 1896 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 859.8928833007812, "completions/mean_terminated_length": 752.9559478759766, "completions/min_length": 281.0, "completions/min_terminated_length": 281.0, "epoch": 0.5666492420282279, "grad_norm": 0.4683382213115692, "kl": 1.646484375, "learning_rate": 8.521905888703894e-06, "loss": 0.079, "num_tokens": 929540834.0, "reward": 0.5680803805589676, "reward_std": 0.12176499702036381, "rewards/accuracy_reward/mean": 0.08035714318975806, "rewards/accuracy_reward/std": 0.1985420472919941, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05140746245160699, "step": 1897 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 846.4866485595703, "completions/mean_terminated_length": 760.2478790283203, "completions/min_length": 343.75, "completions/min_terminated_length": 343.75, "epoch": 0.5669479501157494, "grad_norm": 0.5519147515296936, "kl": 1.94140625, "learning_rate": 8.511118280762851e-06, "loss": 0.1107, "num_tokens": 929999228.0, "reward": 0.666294664144516, "reward_std": 0.13796598836779594, "rewards/accuracy_reward/mean": 0.1785714253783226, "rewards/accuracy_reward/std": 0.37549861520528793, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 1898 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 849.482177734375, "completions/mean_terminated_length": 752.4755554199219, "completions/min_length": 411.25, "completions/min_terminated_length": 411.25, "epoch": 0.5672466582032708, "grad_norm": 0.5467572808265686, "kl": 1.7783203125, "learning_rate": 8.50033244445955e-06, "loss": 0.1113, "num_tokens": 930451412.0, "reward": 0.6668526977300644, "reward_std": 0.18932094052433968, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.36662162840366364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04537561582401395, "step": 1899 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35937499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 859.4464569091797, "completions/mean_terminated_length": 767.2642364501953, "completions/min_length": 460.5, "completions/min_terminated_length": 460.5, "epoch": 0.5675453662907923, "grad_norm": 0.2174157351255417, "kl": 1.814453125, "learning_rate": 8.489548392628191e-06, "loss": 0.0846, "num_tokens": 930907660.0, "reward": 0.6679687798023224, "reward_std": 0.17810029163956642, "rewards/accuracy_reward/mean": 0.17633928917348385, "rewards/accuracy_reward/std": 0.3760026767849922, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04293336346745491, "step": 1900 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 874.1986999511719, "completions/mean_terminated_length": 777.1899108886719, "completions/min_length": 408.75, "completions/min_terminated_length": 408.75, "epoch": 0.5678440743783137, "grad_norm": 0.2088894098997116, "kl": 2.328125, "learning_rate": 8.478766138100834e-06, "loss": 0.1148, "num_tokens": 931382005.0, "reward": 0.6774553805589676, "reward_std": 0.13991169072687626, "rewards/accuracy_reward/mean": 0.18749999906867743, "rewards/accuracy_reward/std": 0.3675423935055733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047143861185759306, "step": 1901 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 805.5335083007812, "completions/mean_terminated_length": 717.3806915283203, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "epoch": 0.5681427824658353, "grad_norm": 0.22877703607082367, "kl": 2.96875, "learning_rate": 8.467985693707417e-06, "loss": 0.1768, "num_tokens": 931817268.0, "reward": 0.7087053805589676, "reward_std": 0.2415581662207842, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.41765296459198, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05663948878645897, "step": 1902 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 837.3348388671875, "completions/mean_terminated_length": 756.394775390625, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "epoch": 0.5684414905533567, "grad_norm": 0.24063847959041595, "kl": 1.673828125, "learning_rate": 8.457207072275712e-06, "loss": 0.0915, "num_tokens": 932256154.0, "reward": 0.7003348469734192, "reward_std": 0.11063447641208768, "rewards/accuracy_reward/mean": 0.2075892873108387, "rewards/accuracy_reward/std": 0.39948850125074387, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04205985926091671, "step": 1903 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 852.6964721679688, "completions/mean_terminated_length": 767.7384490966797, "completions/min_length": 336.75, "completions/min_terminated_length": 336.75, "epoch": 0.5687401986408782, "grad_norm": 0.3258196711540222, "kl": 2.060546875, "learning_rate": 8.44643028663133e-06, "loss": 0.1003, "num_tokens": 932706658.0, "reward": 0.5496651977300644, "reward_std": 0.09978253208100796, "rewards/accuracy_reward/mean": 0.06138392933644354, "rewards/accuracy_reward/std": 0.22114940546453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04334343643859029, "step": 1904 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 862.5357513427734, "completions/mean_terminated_length": 779.0154113769531, "completions/min_length": 403.5, "completions/min_terminated_length": 403.5, "epoch": 0.5690389067283996, "grad_norm": 0.7077783346176147, "kl": 1.74609375, "learning_rate": 8.43565534959769e-06, "loss": 0.1017, "num_tokens": 933163138.0, "reward": 0.6875000298023224, "reward_std": 0.19941462948918343, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.39121198654174805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.028279099613428116, "step": 1905 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 855.185302734375, "completions/mean_terminated_length": 785.4850158691406, "completions/min_length": 435.5, "completions/min_terminated_length": 435.5, "epoch": 0.5693376148159212, "grad_norm": 0.2914312481880188, "kl": 1.37890625, "learning_rate": 8.424882273996023e-06, "loss": 0.0718, "num_tokens": 933621685.0, "reward": 0.6344866454601288, "reward_std": 0.1179117620922625, "rewards/accuracy_reward/mean": 0.1383928544819355, "rewards/accuracy_reward/std": 0.2868412435054779, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 1906 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.30357142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 848.7857513427734, "completions/mean_terminated_length": 773.1343383789062, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "epoch": 0.5696363229034426, "grad_norm": 0.20548272132873535, "kl": 0.845703125, "learning_rate": 8.41411107264533e-06, "loss": 0.049, "num_tokens": 934076533.0, "reward": 0.6612723469734192, "reward_std": 0.11201489344239235, "rewards/accuracy_reward/mean": 0.1629464253783226, "rewards/accuracy_reward/std": 0.36811886727809906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4983258992433548, "rewards/tag_count_reward/std": 0.017717084381729364, "step": 1907 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 891.2277069091797, "completions/mean_terminated_length": 808.7820587158203, "completions/min_length": 462.75, "completions/min_terminated_length": 462.75, "epoch": 0.5699350309909641, "grad_norm": 0.9116973876953125, "kl": 1.671875, "learning_rate": 8.403341758362391e-06, "loss": 0.0952, "num_tokens": 934543739.0, "reward": 0.6277902126312256, "reward_std": 0.19127531722187996, "rewards/accuracy_reward/mean": 0.13616071501746774, "rewards/accuracy_reward/std": 0.3116225190460682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04288960574194789, "step": 1908 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 876.9911193847656, "completions/mean_terminated_length": 783.6461944580078, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 0.5702337390784855, "grad_norm": 0.33412861824035645, "kl": 1.48828125, "learning_rate": 8.392574343961739e-06, "loss": 0.0729, "num_tokens": 935013175.0, "reward": 0.6640625447034836, "reward_std": 0.1416482999920845, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.3731291815638542, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 1909 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 840.8616485595703, "completions/mean_terminated_length": 754.8844909667969, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "epoch": 0.570532447166007, "grad_norm": 0.24821510910987854, "kl": 0.97265625, "learning_rate": 8.38180884225565e-06, "loss": 0.0591, "num_tokens": 935464041.0, "reward": 0.7483259290456772, "reward_std": 0.23848251067101955, "rewards/accuracy_reward/mean": 0.2522321455180645, "rewards/accuracy_reward/std": 0.4010177552700043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1910 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 859.9397735595703, "completions/mean_terminated_length": 754.4132995605469, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.5708311552535285, "grad_norm": 0.494156152009964, "kl": 1.74609375, "learning_rate": 8.371045266054114e-06, "loss": 0.1021, "num_tokens": 935924702.0, "reward": 0.5703125223517418, "reward_std": 0.10914112720638514, "rewards/accuracy_reward/mean": 0.0829613097012043, "rewards/accuracy_reward/std": 0.23365361243486404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 1911 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 879.5625457763672, "completions/mean_terminated_length": 783.7828216552734, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.57112986334105, "grad_norm": 0.1461511254310608, "kl": 1.0439453125, "learning_rate": 8.36028362816484e-06, "loss": 0.0599, "num_tokens": 936396426.0, "reward": 0.6210937798023224, "reward_std": 0.1672674287110567, "rewards/accuracy_reward/mean": 0.12499999906867743, "rewards/accuracy_reward/std": 0.32042286545038223, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870585348457098, "step": 1912 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 885.6116333007812, "completions/mean_terminated_length": 780.922607421875, "completions/min_length": 465.0, "completions/min_terminated_length": 465.0, "epoch": 0.5714285714285714, "grad_norm": 0.17571422457695007, "kl": 0.9970703125, "learning_rate": 8.349523941393224e-06, "loss": 0.0513, "num_tokens": 936863932.0, "reward": 0.6891741454601288, "reward_std": 0.1725537721067667, "rewards/accuracy_reward/mean": 0.1941964328289032, "rewards/accuracy_reward/std": 0.3857019171118736, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.033598463982343674, "step": 1913 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 878.7031555175781, "completions/mean_terminated_length": 789.3950042724609, "completions/min_length": 419.75, "completions/min_terminated_length": 419.75, "epoch": 0.5717272795160929, "grad_norm": 0.12959812581539154, "kl": 1.119140625, "learning_rate": 8.338766218542348e-06, "loss": 0.0605, "num_tokens": 937333031.0, "reward": 0.7907366454601288, "reward_std": 0.17039262130856514, "rewards/accuracy_reward/mean": 0.3225446529686451, "rewards/accuracy_reward/std": 0.4581467881798744, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03752126870676875, "step": 1914 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 865.5670013427734, "completions/mean_terminated_length": 764.1754150390625, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.5720259876036143, "grad_norm": 0.18719466030597687, "kl": 1.0006103515625, "learning_rate": 8.328010472412943e-06, "loss": 0.0404, "num_tokens": 937792997.0, "reward": 0.6121652275323868, "reward_std": 0.13536064513027668, "rewards/accuracy_reward/mean": 0.11607142677530646, "rewards/accuracy_reward/std": 0.288056094199419, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 1915 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 892.310302734375, "completions/mean_terminated_length": 774.8664245605469, "completions/min_length": 401.25, "completions/min_terminated_length": 401.25, "epoch": 0.5723246956911359, "grad_norm": 0.33191928267478943, "kl": 0.8701171875, "learning_rate": 8.317256715803407e-06, "loss": 0.0516, "num_tokens": 938266608.0, "reward": 0.6757812649011612, "reward_std": 0.1585485376417637, "rewards/accuracy_reward/mean": 0.1845238097012043, "rewards/accuracy_reward/std": 0.3692430928349495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.03309101238846779, "step": 1916 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 905.6406707763672, "completions/mean_terminated_length": 798.383544921875, "completions/min_length": 468.75, "completions/min_terminated_length": 468.75, "epoch": 0.5726234037786573, "grad_norm": 0.15385296940803528, "kl": 1.0576171875, "learning_rate": 8.306504961509755e-06, "loss": 0.0468, "num_tokens": 938741343.0, "reward": 0.6841518133878708, "reward_std": 0.1490306481719017, "rewards/accuracy_reward/mean": 0.20014880783855915, "rewards/accuracy_reward/std": 0.3912811279296875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 1917 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5691964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 917.4420166015625, "completions/mean_terminated_length": 779.0205078125, "completions/min_length": 420.75, "completions/min_terminated_length": 420.75, "epoch": 0.5729221118661788, "grad_norm": 0.2443465292453766, "kl": 0.634765625, "learning_rate": 8.295755222325625e-06, "loss": 0.0339, "num_tokens": 939230325.0, "reward": 0.5608259290456772, "reward_std": 0.07528881821781397, "rewards/accuracy_reward/mean": 0.06473214481957257, "rewards/accuracy_reward/std": 0.19886966235935688, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 1918 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 907.6652221679688, "completions/mean_terminated_length": 797.893310546875, "completions/min_length": 288.25, "completions/min_terminated_length": 288.25, "epoch": 0.5732208199537002, "grad_norm": 0.14420664310455322, "kl": 0.7548828125, "learning_rate": 8.285007511042259e-06, "loss": 0.0477, "num_tokens": 939711247.0, "reward": 0.6255580633878708, "reward_std": 0.11168921925127506, "rewards/accuracy_reward/mean": 0.12946428847499192, "rewards/accuracy_reward/std": 0.30394077859818935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937425494194, "rewards/tag_count_reward/std": 0.030848319176584482, "step": 1919 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 894.6295013427734, "completions/mean_terminated_length": 784.4563140869141, "completions/min_length": 443.5, "completions/min_terminated_length": 443.5, "epoch": 0.5735195280412217, "grad_norm": 0.26918143033981323, "kl": 1.359375, "learning_rate": 8.274261840448486e-06, "loss": 0.0775, "num_tokens": 940182201.0, "reward": 0.6757812798023224, "reward_std": 0.1557978056371212, "rewards/accuracy_reward/mean": 0.1830357164144516, "rewards/accuracy_reward/std": 0.3652106523513794, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 1920 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 893.513427734375, "completions/mean_terminated_length": 738.5831451416016, "completions/min_length": 363.25, "completions/min_terminated_length": 363.25, "epoch": 0.5738182361287432, "grad_norm": 0.445139080286026, "kl": 1.35546875, "learning_rate": 8.263518223330698e-06, "loss": 0.0801, "num_tokens": 940662223.0, "reward": 0.6378348469734192, "reward_std": 0.14048010483384132, "rewards/accuracy_reward/mean": 0.1473214253783226, "rewards/accuracy_reward/std": 0.35460542142391205, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241046637297, "step": 1921 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6227678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 942.5781555175781, "completions/mean_terminated_length": 818.2192535400391, "completions/min_length": 467.0, "completions/min_terminated_length": 467.0, "epoch": 0.5741169442162647, "grad_norm": 0.32375577092170715, "kl": 1.744140625, "learning_rate": 8.252776672472856e-06, "loss": 0.0922, "num_tokens": 941155058.0, "reward": 0.6171875298023224, "reward_std": 0.18267756327986717, "rewards/accuracy_reward/mean": 0.1294642877765, "rewards/accuracy_reward/std": 0.29905766621232033, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05349547974765301, "step": 1922 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.578125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 929.4353179931641, "completions/mean_terminated_length": 802.1873321533203, "completions/min_length": 507.0, "completions/min_terminated_length": 507.0, "epoch": 0.5744156523037861, "grad_norm": 0.22799600660800934, "kl": 1.2099609375, "learning_rate": 8.242037200656455e-06, "loss": 0.0626, "num_tokens": 941646133.0, "reward": 0.5970982313156128, "reward_std": 0.09545394266024232, "rewards/accuracy_reward/mean": 0.1086309552192688, "rewards/accuracy_reward/std": 0.26458486169576645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697824731469, "step": 1923 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 947.7545013427734, "completions/mean_terminated_length": 779.7785491943359, "completions/min_length": 395.75, "completions/min_terminated_length": 395.75, "epoch": 0.5747143603913076, "grad_norm": 0.3209523856639862, "kl": 1.63671875, "learning_rate": 8.231299820660523e-06, "loss": 0.0844, "num_tokens": 942132839.0, "reward": 0.6796875298023224, "reward_std": 0.16390930116176605, "rewards/accuracy_reward/mean": 0.18973213923163712, "rewards/accuracy_reward/std": 0.34196675196290016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04923219420015812, "step": 1924 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6004464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 936.2589721679688, "completions/mean_terminated_length": 799.3354187011719, "completions/min_length": 513.5, "completions/min_terminated_length": 513.5, "epoch": 0.575013068478829, "grad_norm": 0.2879107892513275, "kl": 2.0947265625, "learning_rate": 8.220564545261585e-06, "loss": 0.11, "num_tokens": 942622539.0, "reward": 0.7070312798023224, "reward_std": 0.24111679941415787, "rewards/accuracy_reward/mean": 0.2187499962747097, "rewards/accuracy_reward/std": 0.40230692923069, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05078771058470011, "step": 1925 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5669642857142858, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 902.3929138183594, "completions/mean_terminated_length": 751.3888397216797, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "epoch": 0.5753117765663506, "grad_norm": 0.2510192096233368, "kl": 2.37109375, "learning_rate": 8.209831387233675e-06, "loss": 0.1333, "num_tokens": 943097691.0, "reward": 0.726004496216774, "reward_std": 0.17518112435936928, "rewards/accuracy_reward/mean": 0.238839291036129, "rewards/accuracy_reward/std": 0.38298580050468445, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055374542251229286, "step": 1926 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 898.9620819091797, "completions/mean_terminated_length": 756.5210266113281, "completions/min_length": 450.25, "completions/min_terminated_length": 450.25, "epoch": 0.575610484653872, "grad_norm": 0.39368852972984314, "kl": 2.23828125, "learning_rate": 8.199100359348303e-06, "loss": 0.1289, "num_tokens": 943572410.0, "reward": 0.7315848618745804, "reward_std": 0.21148592978715897, "rewards/accuracy_reward/mean": 0.24330356903374195, "rewards/accuracy_reward/std": 0.40684883296489716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 1927 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.7098214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 956.0112152099609, "completions/mean_terminated_length": 797.4808044433594, "completions/min_length": 477.5, "completions/min_terminated_length": 477.5, "epoch": 0.5759091927413935, "grad_norm": 0.31229400634765625, "kl": 2.318359375, "learning_rate": 8.188371474374448e-06, "loss": 0.1113, "num_tokens": 944075391.0, "reward": 0.5837053805589676, "reward_std": 0.1627472434192896, "rewards/accuracy_reward/mean": 0.0959821417927742, "rewards/accuracy_reward/std": 0.27765416353940964, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05409308150410652, "step": 1928 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 930.6495971679688, "completions/mean_terminated_length": 798.0955200195312, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "epoch": 0.5762079008289149, "grad_norm": 0.39204227924346924, "kl": 3.0703125, "learning_rate": 8.177644745078525e-06, "loss": 0.1365, "num_tokens": 944578034.0, "reward": 0.5563616305589676, "reward_std": 0.13480405882000923, "rewards/accuracy_reward/mean": 0.0736607164144516, "rewards/accuracy_reward/std": 0.2291383445262909, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06362892128527164, "step": 1929 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6294642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 933.0670013427734, "completions/mean_terminated_length": 785.4864196777344, "completions/min_length": 444.75, "completions/min_terminated_length": 444.75, "epoch": 0.5765066089164365, "grad_norm": 0.6350332498550415, "kl": 2.89453125, "learning_rate": 8.166920184224409e-06, "loss": 0.1436, "num_tokens": 945068080.0, "reward": 0.6395089477300644, "reward_std": 0.18348483368754387, "rewards/accuracy_reward/mean": 0.1540178544819355, "rewards/accuracy_reward/std": 0.3291335701942444, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058577682822942734, "step": 1930 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.640625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 935.3080749511719, "completions/mean_terminated_length": 780.9713134765625, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "epoch": 0.5768053170039579, "grad_norm": 0.5970727801322937, "kl": 3.44140625, "learning_rate": 8.156197804573368e-06, "loss": 0.153, "num_tokens": 945562138.0, "reward": 0.5731027126312256, "reward_std": 0.15868965908885002, "rewards/accuracy_reward/mean": 0.09151785774156451, "rewards/accuracy_reward/std": 0.2664184682071209, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848171710968, "rewards/tag_count_reward/std": 0.0652739554643631, "step": 1931 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6897321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 951.8036041259766, "completions/mean_terminated_length": 790.6859283447266, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "epoch": 0.5771040250914794, "grad_norm": 0.604433536529541, "kl": 2.9765625, "learning_rate": 8.145477618884092e-06, "loss": 0.1359, "num_tokens": 946065010.0, "reward": 0.5820312798023224, "reward_std": 0.1513807736337185, "rewards/accuracy_reward/mean": 0.09598214412108064, "rewards/accuracy_reward/std": 0.27454717829823494, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.055549753829836845, "step": 1932 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5915178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 903.0960235595703, "completions/mean_terminated_length": 742.0120849609375, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.5774027331790008, "grad_norm": 0.4448097050189972, "kl": 2.6953125, "learning_rate": 8.134759639912655e-06, "loss": 0.1459, "num_tokens": 946533837.0, "reward": 0.663504496216774, "reward_std": 0.1649934183806181, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.37793517112731934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05500977020710707, "step": 1933 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 904.5692291259766, "completions/mean_terminated_length": 759.0304565429688, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.5777014412665223, "grad_norm": 0.29537805914878845, "kl": 2.208984375, "learning_rate": 8.124043880412506e-06, "loss": 0.1229, "num_tokens": 947005148.0, "reward": 0.6696428805589676, "reward_std": 0.20597128942608833, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.3095366880297661, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 1934 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 906.6116485595703, "completions/mean_terminated_length": 762.7791595458984, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.5780001493540438, "grad_norm": 0.2868776321411133, "kl": 2.28515625, "learning_rate": 8.11333035313445e-06, "loss": 0.1169, "num_tokens": 947481886.0, "reward": 0.6936384290456772, "reward_std": 0.16249815560877323, "rewards/accuracy_reward/mean": 0.21205356996506453, "rewards/accuracy_reward/std": 0.3606190159916878, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06488452106714249, "step": 1935 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6205357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 937.1585235595703, "completions/mean_terminated_length": 800.933837890625, "completions/min_length": 423.75, "completions/min_terminated_length": 423.75, "epoch": 0.5782988574415653, "grad_norm": 0.6509236097335815, "kl": 1.943359375, "learning_rate": 8.10261907082664e-06, "loss": 0.1062, "num_tokens": 947982165.0, "reward": 0.6378348618745804, "reward_std": 0.14845769200474024, "rewards/accuracy_reward/mean": 0.1595982164144516, "rewards/accuracy_reward/std": 0.3074847608804703, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05724119208753109, "step": 1936 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.671875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 961.4152221679688, "completions/mean_terminated_length": 840.4158020019531, "completions/min_length": 428.5, "completions/min_terminated_length": 428.5, "epoch": 0.5785975655290867, "grad_norm": 0.33733582496643066, "kl": 2.169921875, "learning_rate": 8.091910046234552e-06, "loss": 0.109, "num_tokens": 948487343.0, "reward": 0.6004464626312256, "reward_std": 0.1491569634526968, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.3094189167022705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.06228757090866566, "step": 1937 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 909.950927734375, "completions/mean_terminated_length": 791.4524688720703, "completions/min_length": 457.25, "completions/min_terminated_length": 457.25, "epoch": 0.5788962736166081, "grad_norm": 0.27086201310157776, "kl": 1.380859375, "learning_rate": 8.08120329210099e-06, "loss": 0.0723, "num_tokens": 948963641.0, "reward": 0.6400669813156128, "reward_std": 0.13836304657161236, "rewards/accuracy_reward/mean": 0.14955357275903225, "rewards/accuracy_reward/std": 0.3419748395681381, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 1938 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 883.8326263427734, "completions/mean_terminated_length": 745.6858825683594, "completions/min_length": 396.5, "completions/min_terminated_length": 396.5, "epoch": 0.5791949817041296, "grad_norm": 0.2709704339504242, "kl": 1.8203125, "learning_rate": 8.070498821166031e-06, "loss": 0.0949, "num_tokens": 949435310.0, "reward": 0.6411830633878708, "reward_std": 0.15804519318044186, "rewards/accuracy_reward/mean": 0.15401786100119352, "rewards/accuracy_reward/std": 0.3349352926015854, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05416911095380783, "step": 1939 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 882.8281555175781, "completions/mean_terminated_length": 761.9947052001953, "completions/min_length": 308.25, "completions/min_terminated_length": 308.25, "epoch": 0.5794936897916511, "grad_norm": 0.5985023379325867, "kl": 1.966796875, "learning_rate": 8.059796646167062e-06, "loss": 0.1126, "num_tokens": 949910753.0, "reward": 0.6768973618745804, "reward_std": 0.18206359539180994, "rewards/accuracy_reward/mean": 0.19196428451687098, "rewards/accuracy_reward/std": 0.36356277018785477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05834365636110306, "step": 1940 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 902.4420013427734, "completions/mean_terminated_length": 774.2493591308594, "completions/min_length": 395.5, "completions/min_terminated_length": 395.5, "epoch": 0.5797923978791726, "grad_norm": 0.4559721350669861, "kl": 1.701171875, "learning_rate": 8.04909677983872e-06, "loss": 0.0901, "num_tokens": 950377895.0, "reward": 0.6250000447034836, "reward_std": 0.18107835575938225, "rewards/accuracy_reward/mean": 0.13839285913854837, "rewards/accuracy_reward/std": 0.3188636526465416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05536533612757921, "step": 1941 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45758928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 891.3348541259766, "completions/mean_terminated_length": 790.5011444091797, "completions/min_length": 469.75, "completions/min_terminated_length": 469.75, "epoch": 0.580091105966694, "grad_norm": 0.3433072566986084, "kl": 2.515625, "learning_rate": 8.038399234912899e-06, "loss": 0.1481, "num_tokens": 950854525.0, "reward": 0.72433041036129, "reward_std": 0.23017451912164688, "rewards/accuracy_reward/mean": 0.23883928544819355, "rewards/accuracy_reward/std": 0.3824949562549591, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.0578513452783227, "step": 1942 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 887.6228179931641, "completions/mean_terminated_length": 783.5630340576172, "completions/min_length": 418.5, "completions/min_terminated_length": 418.5, "epoch": 0.5803898140542155, "grad_norm": 0.4738178253173828, "kl": 2.359375, "learning_rate": 8.027704024118742e-06, "loss": 0.1392, "num_tokens": 951325380.0, "reward": 0.5954241305589676, "reward_std": 0.17253964766860008, "rewards/accuracy_reward/mean": 0.11160714295692742, "rewards/accuracy_reward/std": 0.2893621977418661, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.061596741899847984, "step": 1943 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 905.9620971679688, "completions/mean_terminated_length": 799.7131500244141, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.5806885221417369, "grad_norm": 0.33211690187454224, "kl": 2.626953125, "learning_rate": 8.017011160182594e-06, "loss": 0.1318, "num_tokens": 951804099.0, "reward": 0.5976562649011612, "reward_std": 0.17725927010178566, "rewards/accuracy_reward/mean": 0.11383928777649999, "rewards/accuracy_reward/std": 0.284579623490572, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169515132904, "rewards/tag_count_reward/std": 0.05951428692787886, "step": 1944 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 896.9308319091797, "completions/mean_terminated_length": 814.0224609375, "completions/min_length": 415.75, "completions/min_terminated_length": 415.75, "epoch": 0.5809872302292585, "grad_norm": 0.34970834851264954, "kl": 2.197265625, "learning_rate": 8.00632065582803e-06, "loss": 0.1162, "num_tokens": 952274468.0, "reward": 0.6372768133878708, "reward_std": 0.1685448158532381, "rewards/accuracy_reward/mean": 0.1473214328289032, "rewards/accuracy_reward/std": 0.34523439407348633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04923219420015812, "step": 1945 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2946428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 817.5402221679688, "completions/mean_terminated_length": 733.6586303710938, "completions/min_length": 286.25, "completions/min_terminated_length": 286.25, "epoch": 0.5812859383167799, "grad_norm": 0.23642799258232117, "kl": 2.16015625, "learning_rate": 7.995632523775795e-06, "loss": 0.1397, "num_tokens": 952710518.0, "reward": 0.7059151977300644, "reward_std": 0.16508204117417336, "rewards/accuracy_reward/mean": 0.21428571455180645, "rewards/accuracy_reward/std": 0.37687060981988907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 1946 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 858.1986999511719, "completions/mean_terminated_length": 782.0103759765625, "completions/min_length": 439.5, "completions/min_terminated_length": 439.5, "epoch": 0.5815846464043014, "grad_norm": 0.5545164942741394, "kl": 3.28125, "learning_rate": 7.984946776743829e-06, "loss": 0.1691, "num_tokens": 953163983.0, "reward": 0.5753348469734192, "reward_std": 0.129177859518677, "rewards/accuracy_reward/mean": 0.09449404804036021, "rewards/accuracy_reward/std": 0.23048260062932968, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 1947 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27455357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 791.5647583007812, "completions/mean_terminated_length": 706.5105590820312, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "epoch": 0.5818833544918228, "grad_norm": 1.1458666324615479, "kl": 3.265625, "learning_rate": 7.974263427447225e-06, "loss": 0.1793, "num_tokens": 953588300.0, "reward": 0.728794664144516, "reward_std": 0.23633801564574242, "rewards/accuracy_reward/mean": 0.2388392835855484, "rewards/accuracy_reward/std": 0.42363540828227997, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764227330685, "step": 1948 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 816.1808319091797, "completions/mean_terminated_length": 727.2447052001953, "completions/min_length": 309.25, "completions/min_terminated_length": 309.25, "epoch": 0.5821820625793444, "grad_norm": 0.5728418231010437, "kl": 3.671875, "learning_rate": 7.963582488598227e-06, "loss": 0.1857, "num_tokens": 954029917.0, "reward": 0.6222098469734192, "reward_std": 0.14177008159458637, "rewards/accuracy_reward/mean": 0.13839285587891936, "rewards/accuracy_reward/std": 0.32195228338241577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06072821468114853, "step": 1949 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 834.2053833007812, "completions/mean_terminated_length": 734.8211975097656, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 0.5824807706668658, "grad_norm": 0.36389097571372986, "kl": 2.96484375, "learning_rate": 7.952903972906205e-06, "loss": 0.1526, "num_tokens": 954473657.0, "reward": 0.6729910969734192, "reward_std": 0.1728004775941372, "rewards/accuracy_reward/mean": 0.18526785634458065, "rewards/accuracy_reward/std": 0.3823835328221321, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.054059810005128384, "step": 1950 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 817.6920013427734, "completions/mean_terminated_length": 712.6631011962891, "completions/min_length": 286.25, "completions/min_terminated_length": 286.25, "epoch": 0.5827794787543873, "grad_norm": 0.3425004184246063, "kl": 2.71484375, "learning_rate": 7.942227893077652e-06, "loss": 0.1454, "num_tokens": 954917039.0, "reward": 0.742745578289032, "reward_std": 0.2015436515212059, "rewards/accuracy_reward/mean": 0.2544642873108387, "rewards/accuracy_reward/std": 0.38982153311371803, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 1951 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 802.0469207763672, "completions/mean_terminated_length": 703.8012847900391, "completions/min_length": 319.5, "completions/min_terminated_length": 319.5, "epoch": 0.5830781868419087, "grad_norm": 0.2972990870475769, "kl": 2.517578125, "learning_rate": 7.931554261816159e-06, "loss": 0.1563, "num_tokens": 955352212.0, "reward": 0.7070312798023224, "reward_std": 0.15703326929360628, "rewards/accuracy_reward/mean": 0.2165178619325161, "rewards/accuracy_reward/std": 0.389196552336216, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1952 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 851.2031707763672, "completions/mean_terminated_length": 732.8092346191406, "completions/min_length": 343.75, "completions/min_terminated_length": 343.75, "epoch": 0.5833768949294302, "grad_norm": 0.23336295783519745, "kl": 2.41015625, "learning_rate": 7.92088309182241e-06, "loss": 0.1265, "num_tokens": 955810527.0, "reward": 0.6757812798023224, "reward_std": 0.17414561100304127, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3786157965660095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918293237686, "step": 1953 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 830.3393249511719, "completions/mean_terminated_length": 739.5004119873047, "completions/min_length": 311.75, "completions/min_terminated_length": 311.75, "epoch": 0.5836756030169516, "grad_norm": 0.4374505281448364, "kl": 3.123046875, "learning_rate": 7.910214395794142e-06, "loss": 0.1635, "num_tokens": 956257495.0, "reward": 0.6205357387661934, "reward_std": 0.13171652518212795, "rewards/accuracy_reward/mean": 0.1339285746216774, "rewards/accuracy_reward/std": 0.2853953540325165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 1954 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 886.8683471679688, "completions/mean_terminated_length": 757.227294921875, "completions/min_length": 322.75, "completions/min_terminated_length": 322.75, "epoch": 0.5839743111044732, "grad_norm": 0.3197365701198578, "kl": 2.626953125, "learning_rate": 7.899548186426177e-06, "loss": 0.1423, "num_tokens": 956725084.0, "reward": 0.688058078289032, "reward_std": 0.20089598558843136, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.39464861899614334, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.0587494820356369, "step": 1955 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 873.1272583007812, "completions/mean_terminated_length": 769.2339324951172, "completions/min_length": 378.5, "completions/min_terminated_length": 378.5, "epoch": 0.5842730191919946, "grad_norm": 0.14266656339168549, "kl": 1.33984375, "learning_rate": 7.888884476410348e-06, "loss": 0.0659, "num_tokens": 957185925.0, "reward": 0.5725446492433548, "reward_std": 0.07802437990903854, "rewards/accuracy_reward/mean": 0.07812499813735485, "rewards/accuracy_reward/std": 0.22852876037359238, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 1956 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 853.9263763427734, "completions/mean_terminated_length": 734.4948120117188, "completions/min_length": 386.25, "completions/min_terminated_length": 386.25, "epoch": 0.5845717272795161, "grad_norm": 0.21552003920078278, "kl": 1.28515625, "learning_rate": 7.878223278435539e-06, "loss": 0.0689, "num_tokens": 957636132.0, "reward": 0.7438616454601288, "reward_std": 0.14683028496801853, "rewards/accuracy_reward/mean": 0.2499999962747097, "rewards/accuracy_reward/std": 0.4288642108440399, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 1957 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3214285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 800.2589569091797, "completions/mean_terminated_length": 697.8660888671875, "completions/min_length": 236.75, "completions/min_terminated_length": 236.75, "epoch": 0.5848704353670375, "grad_norm": 0.20770277082920074, "kl": 1.71484375, "learning_rate": 7.867564605187625e-06, "loss": 0.1115, "num_tokens": 958077080.0, "reward": 0.6383928805589676, "reward_std": 0.17490672320127487, "rewards/accuracy_reward/mean": 0.14694940391927958, "rewards/accuracy_reward/std": 0.3420775271952152, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03410126641392708, "step": 1958 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46205357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 865.4152221679688, "completions/mean_terminated_length": 729.3810882568359, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "epoch": 0.5851691434545591, "grad_norm": 0.4143705666065216, "kl": 1.60546875, "learning_rate": 7.856908469349495e-06, "loss": 0.1045, "num_tokens": 958536898.0, "reward": 0.742745578289032, "reward_std": 0.19772985391318798, "rewards/accuracy_reward/mean": 0.2522321417927742, "rewards/accuracy_reward/std": 0.3926466479897499, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 1959 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.75, "completions/mean_length": 871.2522583007812, "completions/mean_terminated_length": 741.9913177490234, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.5854678515420805, "grad_norm": 0.5400915145874023, "kl": 1.603515625, "learning_rate": 7.846254883601e-06, "loss": 0.09, "num_tokens": 958995107.0, "reward": 0.6344866305589676, "reward_std": 0.12227210029959679, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.34197157248854637, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 1960 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5647321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 900.6562957763672, "completions/mean_terminated_length": 739.1165924072266, "completions/min_length": 435.25, "completions/min_terminated_length": 435.25, "epoch": 0.585766559629602, "grad_norm": 0.1762644201517105, "kl": 1.447265625, "learning_rate": 7.835603860618973e-06, "loss": 0.0732, "num_tokens": 959471065.0, "reward": 0.655691996216774, "reward_std": 0.1542899664491415, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.35507889091968536, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 1961 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 874.4442443847656, "completions/mean_terminated_length": 745.5226440429688, "completions/min_length": 403.25, "completions/min_terminated_length": 403.25, "epoch": 0.5860652677171234, "grad_norm": 0.21655777096748352, "kl": 1.7216796875, "learning_rate": 7.824955413077185e-06, "loss": 0.1007, "num_tokens": 959935904.0, "reward": 0.7187500298023224, "reward_std": 0.20437804609537125, "rewards/accuracy_reward/mean": 0.2276785746216774, "rewards/accuracy_reward/std": 0.3757499009370804, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194977223873, "step": 1962 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 851.2723541259766, "completions/mean_terminated_length": 658.8264465332031, "completions/min_length": 294.75, "completions/min_terminated_length": 294.75, "epoch": 0.5863639758046449, "grad_norm": 0.3501463532447815, "kl": 1.5703125, "learning_rate": 7.814309553646357e-06, "loss": 0.0808, "num_tokens": 960385514.0, "reward": 0.681919664144516, "reward_std": 0.11045054905116558, "rewards/accuracy_reward/mean": 0.1897321380674839, "rewards/accuracy_reward/std": 0.38550107926130295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 1963 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 825.2366333007812, "completions/mean_terminated_length": 706.5472869873047, "completions/min_length": 331.25, "completions/min_terminated_length": 331.25, "epoch": 0.5866626838921664, "grad_norm": 0.29765933752059937, "kl": 1.103515625, "learning_rate": 7.803666294994111e-06, "loss": 0.0566, "num_tokens": 960827748.0, "reward": 0.702566996216774, "reward_std": 0.13275071419775486, "rewards/accuracy_reward/mean": 0.20758928637951612, "rewards/accuracy_reward/std": 0.37910646945238113, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 1964 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 843.7120819091797, "completions/mean_terminated_length": 706.1165161132812, "completions/min_length": 301.25, "completions/min_terminated_length": 301.25, "epoch": 0.5869613919796879, "grad_norm": 0.4309239685535431, "kl": 1.65234375, "learning_rate": 7.793025649784991e-06, "loss": 0.0812, "num_tokens": 961271843.0, "reward": 0.6986607313156128, "reward_std": 0.1601854059845209, "rewards/accuracy_reward/mean": 0.20535713993012905, "rewards/accuracy_reward/std": 0.39565591514110565, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 1965 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.25, "completions/mean_length": 825.1094055175781, "completions/mean_terminated_length": 714.8127288818359, "completions/min_length": 308.5, "completions/min_terminated_length": 308.5, "epoch": 0.5872601000672093, "grad_norm": 0.20803606510162354, "kl": 1.259765625, "learning_rate": 7.782387630680422e-06, "loss": 0.078, "num_tokens": 961711924.0, "reward": 0.584263414144516, "reward_std": 0.09044557111337781, "rewards/accuracy_reward/mean": 0.0892857164144516, "rewards/accuracy_reward/std": 0.23438546806573868, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.034184794407337904, "step": 1966 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 799.1451263427734, "completions/mean_terminated_length": 702.0712432861328, "completions/min_length": 311.25, "completions/min_terminated_length": 311.25, "epoch": 0.5875588081547308, "grad_norm": 0.5138236284255981, "kl": 1.6796875, "learning_rate": 7.771752250338712e-06, "loss": 0.0939, "num_tokens": 962138693.0, "reward": 0.78683041036129, "reward_std": 0.2023376040160656, "rewards/accuracy_reward/mean": 0.3098958395421505, "rewards/accuracy_reward/std": 0.4498230442404747, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04923219420015812, "step": 1967 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 872.5312957763672, "completions/mean_terminated_length": 738.2674560546875, "completions/min_length": 313.75, "completions/min_terminated_length": 313.75, "epoch": 0.5878575162422522, "grad_norm": 0.48903587460517883, "kl": 1.68359375, "learning_rate": 7.761119521415017e-06, "loss": 0.1043, "num_tokens": 962604019.0, "reward": 0.6227678880095482, "reward_std": 0.0768018551170826, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.27260245382785797, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03946960438042879, "step": 1968 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 880.9576263427734, "completions/mean_terminated_length": 752.5923767089844, "completions/min_length": 392.75, "completions/min_terminated_length": 392.75, "epoch": 0.5881562243297738, "grad_norm": 0.18988437950611115, "kl": 1.1064453125, "learning_rate": 7.750489456561351e-06, "loss": 0.0579, "num_tokens": 963080736.0, "reward": 0.6171875298023224, "reward_std": 0.11223270976915956, "rewards/accuracy_reward/mean": 0.12053571362048388, "rewards/accuracy_reward/std": 0.24852818995714188, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 1969 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 859.3192291259766, "completions/mean_terminated_length": 754.9541473388672, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "epoch": 0.5884549324172952, "grad_norm": 0.17526227235794067, "kl": 0.9794921875, "learning_rate": 7.73986206842655e-06, "loss": 0.0492, "num_tokens": 963537631.0, "reward": 0.5597098469734192, "reward_std": 0.06354673393070698, "rewards/accuracy_reward/mean": 0.06473214225843549, "rewards/accuracy_reward/std": 0.19375306740403175, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.030101283453404903, "step": 1970 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 868.7054138183594, "completions/mean_terminated_length": 747.7571563720703, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "epoch": 0.5887536405048167, "grad_norm": 0.12712979316711426, "kl": 0.78125, "learning_rate": 7.72923736965627e-06, "loss": 0.0403, "num_tokens": 964001707.0, "reward": 0.606584832072258, "reward_std": 0.10100846295244992, "rewards/accuracy_reward/mean": 0.11309524066746235, "rewards/accuracy_reward/std": 0.2660894840955734, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098246216774, "rewards/tag_count_reward/std": 0.026031292509287596, "step": 1971 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 853.3861846923828, "completions/mean_terminated_length": 770.9665985107422, "completions/min_length": 424.25, "completions/min_terminated_length": 424.25, "epoch": 0.5890523485923381, "grad_norm": 0.4027767479419708, "kl": 1.4931640625, "learning_rate": 7.718615372892959e-06, "loss": 0.0864, "num_tokens": 964455416.0, "reward": 0.7070312798023224, "reward_std": 0.15825108252465725, "rewards/accuracy_reward/mean": 0.23028273251838982, "rewards/accuracy_reward/std": 0.35896713845431805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03865890856832266, "step": 1972 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 887.8326263427734, "completions/mean_terminated_length": 787.0114898681641, "completions/min_length": 397.25, "completions/min_terminated_length": 397.25, "epoch": 0.5893510566798597, "grad_norm": 0.17792131006717682, "kl": 1.5517578125, "learning_rate": 7.70799609077586e-06, "loss": 0.0947, "num_tokens": 964927197.0, "reward": 0.6707589477300644, "reward_std": 0.15275930613279343, "rewards/accuracy_reward/mean": 0.17633928172290325, "rewards/accuracy_reward/std": 0.35848304629325867, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 1973 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 836.0379791259766, "completions/mean_terminated_length": 732.0028533935547, "completions/min_length": 435.75, "completions/min_terminated_length": 435.75, "epoch": 0.5896497647673811, "grad_norm": 0.19011445343494415, "kl": 1.826171875, "learning_rate": 7.697379535940978e-06, "loss": 0.0921, "num_tokens": 965374094.0, "reward": 0.6534598469734192, "reward_std": 0.14535938017070293, "rewards/accuracy_reward/mean": 0.16071428451687098, "rewards/accuracy_reward/std": 0.3448670133948326, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.042059858329594135, "step": 1974 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 838.6183471679688, "completions/mean_terminated_length": 759.4110870361328, "completions/min_length": 404.75, "completions/min_terminated_length": 404.75, "epoch": 0.5899484728549026, "grad_norm": 0.5011518001556396, "kl": 2.53125, "learning_rate": 7.686765721021077e-06, "loss": 0.133, "num_tokens": 965819123.0, "reward": 0.6417410969734192, "reward_std": 0.12948929332196712, "rewards/accuracy_reward/mean": 0.1715029743500054, "rewards/accuracy_reward/std": 0.3312962017953396, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 1975 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 854.4419860839844, "completions/mean_terminated_length": 778.5907440185547, "completions/min_length": 388.25, "completions/min_terminated_length": 388.25, "epoch": 0.590247180942424, "grad_norm": 0.21768751740455627, "kl": 1.533203125, "learning_rate": 7.676154658645656e-06, "loss": 0.0762, "num_tokens": 966281433.0, "reward": 0.6473214477300644, "reward_std": 0.15952628664672375, "rewards/accuracy_reward/mean": 0.15401786006987095, "rewards/accuracy_reward/std": 0.3542070537805557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 1976 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43080357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 861.1964569091797, "completions/mean_terminated_length": 744.9371032714844, "completions/min_length": 336.5, "completions/min_terminated_length": 336.5, "epoch": 0.5905458890299455, "grad_norm": 0.32120418548583984, "kl": 1.443359375, "learning_rate": 7.66554636144095e-06, "loss": 0.0715, "num_tokens": 966741793.0, "reward": 0.6060268133878708, "reward_std": 0.09518279740586877, "rewards/accuracy_reward/mean": 0.11160713993012905, "rewards/accuracy_reward/std": 0.2622886523604393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 1977 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 889.1540679931641, "completions/mean_terminated_length": 766.6171112060547, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.590844597117467, "grad_norm": 0.23107579350471497, "kl": 1.65234375, "learning_rate": 7.654940842029886e-06, "loss": 0.0788, "num_tokens": 967208614.0, "reward": 0.5887277126312256, "reward_std": 0.12463352642953396, "rewards/accuracy_reward/mean": 0.09895833511836827, "rewards/accuracy_reward/std": 0.2692617531865835, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04205985926091671, "step": 1978 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 867.2924346923828, "completions/mean_terminated_length": 793.5402374267578, "completions/min_length": 408.25, "completions/min_terminated_length": 408.25, "epoch": 0.5911433052049885, "grad_norm": 0.23767659068107605, "kl": 1.435546875, "learning_rate": 7.644338113032101e-06, "loss": 0.0806, "num_tokens": 967674329.0, "reward": 0.7477678954601288, "reward_std": 0.16906991973519325, "rewards/accuracy_reward/mean": 0.2522321417927742, "rewards/accuracy_reward/std": 0.41054923832416534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 1979 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 861.6897735595703, "completions/mean_terminated_length": 771.6769409179688, "completions/min_length": 366.75, "completions/min_terminated_length": 366.75, "epoch": 0.5914420132925099, "grad_norm": 0.1522180140018463, "kl": 1.00439453125, "learning_rate": 7.633738187063901e-06, "loss": 0.0565, "num_tokens": 968131678.0, "reward": 0.6238839626312256, "reward_std": 0.1276870220899582, "rewards/accuracy_reward/mean": 0.13206845335662365, "rewards/accuracy_reward/std": 0.3356604054570198, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.036314870696514845, "step": 1980 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 850.5201263427734, "completions/mean_terminated_length": 751.9457550048828, "completions/min_length": 421.5, "completions/min_terminated_length": 421.5, "epoch": 0.5917407213800313, "grad_norm": 0.43079873919487, "kl": 1.80859375, "learning_rate": 7.623141076738271e-06, "loss": 0.0932, "num_tokens": 968586919.0, "reward": 0.6796875149011612, "reward_std": 0.16432179138064384, "rewards/accuracy_reward/mean": 0.1875000037252903, "rewards/accuracy_reward/std": 0.3553675711154938, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.04197291610762477, "step": 1981 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 902.0670166015625, "completions/mean_terminated_length": 799.3565826416016, "completions/min_length": 389.75, "completions/min_terminated_length": 389.75, "epoch": 0.5920394294675528, "grad_norm": 0.18775023519992828, "kl": 1.533203125, "learning_rate": 7.612546794664828e-06, "loss": 0.0687, "num_tokens": 969062117.0, "reward": 0.5864955633878708, "reward_std": 0.09441876458004117, "rewards/accuracy_reward/mean": 0.09375000139698386, "rewards/accuracy_reward/std": 0.27862129360437393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040314854588359594, "step": 1982 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 852.4844055175781, "completions/mean_terminated_length": 758.7138214111328, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 0.5923381375550743, "grad_norm": 0.19138726592063904, "kl": 1.4453125, "learning_rate": 7.601955353449838e-06, "loss": 0.0697, "num_tokens": 969510190.0, "reward": 0.6746652126312256, "reward_std": 0.1586278099566698, "rewards/accuracy_reward/mean": 0.18303571082651615, "rewards/accuracy_reward/std": 0.3680851608514786, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04508844017982483, "step": 1983 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 883.8772888183594, "completions/mean_terminated_length": 807.6921691894531, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "epoch": 0.5926368456425958, "grad_norm": 0.21783891320228577, "kl": 1.505859375, "learning_rate": 7.591366765696173e-06, "loss": 0.0828, "num_tokens": 969976343.0, "reward": 0.6434151977300644, "reward_std": 0.16832419112324715, "rewards/accuracy_reward/mean": 0.14955356903374195, "rewards/accuracy_reward/std": 0.34943924099206924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 1984 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 891.4420166015625, "completions/mean_terminated_length": 800.8971710205078, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.5929355537301172, "grad_norm": 0.2856783866882324, "kl": 1.4267578125, "learning_rate": 7.580781044003324e-06, "loss": 0.0686, "num_tokens": 970440829.0, "reward": 0.7047991454601288, "reward_std": 0.1534307897090912, "rewards/accuracy_reward/mean": 0.2120535671710968, "rewards/accuracy_reward/std": 0.39347487688064575, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03955313144251704, "step": 1985 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3214285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 834.4442443847656, "completions/mean_terminated_length": 744.5152587890625, "completions/min_length": 374.25, "completions/min_terminated_length": 374.25, "epoch": 0.5932342618176387, "grad_norm": 0.2837671637535095, "kl": 1.884765625, "learning_rate": 7.570198200967363e-06, "loss": 0.1105, "num_tokens": 970886820.0, "reward": 0.647879496216774, "reward_std": 0.14457088708877563, "rewards/accuracy_reward/mean": 0.15625000279396772, "rewards/accuracy_reward/std": 0.2799157574772835, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 1986 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 888.7924499511719, "completions/mean_terminated_length": 797.8873138427734, "completions/min_length": 455.75, "completions/min_terminated_length": 455.75, "epoch": 0.5935329699051601, "grad_norm": 0.37997540831565857, "kl": 1.474609375, "learning_rate": 7.559618249180935e-06, "loss": 0.0747, "num_tokens": 971356007.0, "reward": 0.6551339477300644, "reward_std": 0.17308441549539566, "rewards/accuracy_reward/mean": 0.1651785746216774, "rewards/accuracy_reward/std": 0.36046646535396576, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 1987 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 882.1228179931641, "completions/mean_terminated_length": 784.5561065673828, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.5938316779926817, "grad_norm": 0.18592941761016846, "kl": 1.51171875, "learning_rate": 7.549041201233256e-06, "loss": 0.0802, "num_tokens": 971818638.0, "reward": 0.6389508992433548, "reward_std": 0.1911165826022625, "rewards/accuracy_reward/mean": 0.14955356996506453, "rewards/accuracy_reward/std": 0.3418300524353981, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 1988 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43303571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 862.7969055175781, "completions/mean_terminated_length": 738.5943908691406, "completions/min_length": 339.75, "completions/min_terminated_length": 339.75, "epoch": 0.5941303860802031, "grad_norm": 0.22177183628082275, "kl": 1.802734375, "learning_rate": 7.53846706971007e-06, "loss": 0.0949, "num_tokens": 972281219.0, "reward": 0.6328125298023224, "reward_std": 0.13053618930280209, "rewards/accuracy_reward/mean": 0.14360118750482798, "rewards/accuracy_reward/std": 0.2976515628397465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.048545535653829575, "step": 1989 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 881.763427734375, "completions/mean_terminated_length": 767.7307891845703, "completions/min_length": 423.0, "completions/min_terminated_length": 423.0, "epoch": 0.5944290941677246, "grad_norm": 0.2429751455783844, "kl": 1.5126953125, "learning_rate": 7.527895867193666e-06, "loss": 0.0918, "num_tokens": 972750345.0, "reward": 0.7098214775323868, "reward_std": 0.24861034750938416, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.4006461575627327, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.03946992103010416, "step": 1990 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4955357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 890.4129791259766, "completions/mean_terminated_length": 766.0753173828125, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "epoch": 0.594727802255246, "grad_norm": 0.5500189065933228, "kl": 1.955078125, "learning_rate": 7.5173276062628364e-06, "loss": 0.1094, "num_tokens": 973220402.0, "reward": 0.6199776977300644, "reward_std": 0.16754676401615143, "rewards/accuracy_reward/mean": 0.13169642887078226, "rewards/accuracy_reward/std": 0.2703391779214144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05243533570319414, "step": 1991 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 879.0424499511719, "completions/mean_terminated_length": 752.7319183349609, "completions/min_length": 425.0, "completions/min_terminated_length": 425.0, "epoch": 0.5950265103427675, "grad_norm": 0.20365749299526215, "kl": 1.841796875, "learning_rate": 7.5067622994928855e-06, "loss": 0.0972, "num_tokens": 973677221.0, "reward": 0.7092634290456772, "reward_std": 0.14856146648526192, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.38347262889146805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983776688576, "step": 1992 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41071428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 864.2790679931641, "completions/mean_terminated_length": 772.1614685058594, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "epoch": 0.595325218430289, "grad_norm": 0.3625529408454895, "kl": 2.025390625, "learning_rate": 7.496199959455584e-06, "loss": 0.1095, "num_tokens": 974136450.0, "reward": 0.5636160969734192, "reward_std": 0.12813200429081917, "rewards/accuracy_reward/mean": 0.07366071362048388, "rewards/accuracy_reward/std": 0.24428891390562057, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 1993 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 899.2924499511719, "completions/mean_terminated_length": 770.4092254638672, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "epoch": 0.5956239265178105, "grad_norm": 0.23447971045970917, "kl": 2.369140625, "learning_rate": 7.485640598719197e-06, "loss": 0.1204, "num_tokens": 974612293.0, "reward": 0.5446428954601288, "reward_std": 0.10658396407961845, "rewards/accuracy_reward/mean": 0.05803571385331452, "rewards/accuracy_reward/std": 0.1817315425723791, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05543891713023186, "step": 1994 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 917.5714721679688, "completions/mean_terminated_length": 800.7627258300781, "completions/min_length": 454.25, "completions/min_terminated_length": 454.25, "epoch": 0.5959226346053319, "grad_norm": 0.2775682210922241, "kl": 2.20703125, "learning_rate": 7.4750842298484205e-06, "loss": 0.1095, "num_tokens": 975101237.0, "reward": 0.6177455708384514, "reward_std": 0.11941658891737461, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.27772653102874756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05263457726687193, "step": 1995 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 914.5536041259766, "completions/mean_terminated_length": 787.6347808837891, "completions/min_length": 409.75, "completions/min_terminated_length": 409.75, "epoch": 0.5962213426928534, "grad_norm": 0.31715765595436096, "kl": 2.78125, "learning_rate": 7.4645308654044065e-06, "loss": 0.1305, "num_tokens": 975578381.0, "reward": 0.6434152126312256, "reward_std": 0.17391351610422134, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3684889003634453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.059022306464612484, "step": 1996 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 882.0000457763672, "completions/mean_terminated_length": 771.7200622558594, "completions/min_length": 359.75, "completions/min_terminated_length": 359.75, "epoch": 0.5965200507803748, "grad_norm": 0.30107730627059937, "kl": 3.189453125, "learning_rate": 7.4539805179447234e-06, "loss": 0.162, "num_tokens": 976050029.0, "reward": 0.734933078289032, "reward_std": 0.22531762719154358, "rewards/accuracy_reward/mean": 0.2522321492433548, "rewards/accuracy_reward/std": 0.42698171734809875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06268814951181412, "step": 1997 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5044642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 907.9107513427734, "completions/mean_terminated_length": 791.8308258056641, "completions/min_length": 426.5, "completions/min_terminated_length": 426.5, "epoch": 0.5968187588678964, "grad_norm": 0.4668862819671631, "kl": 2.115234375, "learning_rate": 7.443433200023358e-06, "loss": 0.1006, "num_tokens": 976525365.0, "reward": 0.5390625149011612, "reward_std": 0.12638911418616772, "rewards/accuracy_reward/mean": 0.0546875, "rewards/accuracy_reward/std": 0.1948273628950119, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.046562228351831436, "step": 1998 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 885.8170013427734, "completions/mean_terminated_length": 761.93310546875, "completions/min_length": 427.0, "completions/min_terminated_length": 427.0, "epoch": 0.5971174669554178, "grad_norm": 0.26019608974456787, "kl": 2.73046875, "learning_rate": 7.4328889241906795e-06, "loss": 0.1429, "num_tokens": 976991219.0, "reward": 0.6010044813156128, "reward_std": 0.1293611079454422, "rewards/accuracy_reward/mean": 0.1160714291036129, "rewards/accuracy_reward/std": 0.26550858467817307, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.058642057701945305, "step": 1999 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 837.3460235595703, "completions/mean_terminated_length": 754.9007568359375, "completions/min_length": 315.25, "completions/min_terminated_length": 315.25, "epoch": 0.5974161750429393, "grad_norm": 0.25564226508140564, "kl": 1.70703125, "learning_rate": 7.4223477029934524e-06, "loss": 0.0856, "num_tokens": 977435246.0, "reward": 0.7070312798023224, "reward_std": 0.18531465157866478, "rewards/accuracy_reward/mean": 0.2165178540162742, "rewards/accuracy_reward/std": 0.35964417457580566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047717904672026634, "step": 2000 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 886.6071929931641, "completions/mean_terminated_length": 770.6021575927734, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "epoch": 0.5977148831304607, "grad_norm": 0.2052658200263977, "kl": 1.837890625, "learning_rate": 7.411809548974792e-06, "loss": 0.1026, "num_tokens": 977899054.0, "reward": 0.733816996216774, "reward_std": 0.16449831426143646, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.42377816140651703, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04597289999946952, "step": 2001 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 845.5111999511719, "completions/mean_terminated_length": 715.1977691650391, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "epoch": 0.5980135912179823, "grad_norm": 0.24894297122955322, "kl": 2.6796875, "learning_rate": 7.401274474674179e-06, "loss": 0.1357, "num_tokens": 978349555.0, "reward": 0.658482164144516, "reward_std": 0.17403878085315228, "rewards/accuracy_reward/mean": 0.17410714365541935, "rewards/accuracy_reward/std": 0.37625957280397415, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.06031146086752415, "step": 2002 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 881.6897735595703, "completions/mean_terminated_length": 767.3505706787109, "completions/min_length": 335.25, "completions/min_terminated_length": 335.25, "epoch": 0.5983122993055037, "grad_norm": 0.5284656882286072, "kl": 2.037109375, "learning_rate": 7.3907424926274115e-06, "loss": 0.1178, "num_tokens": 978818232.0, "reward": 0.7109375447034836, "reward_std": 0.16489035822451115, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.4063519909977913, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 2003 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 879.7589721679688, "completions/mean_terminated_length": 766.0430908203125, "completions/min_length": 368.0, "completions/min_terminated_length": 368.0, "epoch": 0.5986110073930252, "grad_norm": 0.5380033254623413, "kl": 2.298828125, "learning_rate": 7.380213615366627e-06, "loss": 0.1261, "num_tokens": 979281740.0, "reward": 0.6188616305589676, "reward_std": 0.15182428993284702, "rewards/accuracy_reward/mean": 0.1339285708963871, "rewards/accuracy_reward/std": 0.3335573896765709, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05911104939877987, "step": 2004 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 842.6451110839844, "completions/mean_terminated_length": 717.7378997802734, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "epoch": 0.5989097154805466, "grad_norm": 0.32179293036460876, "kl": 2.09375, "learning_rate": 7.3696878554202525e-06, "loss": 0.124, "num_tokens": 979729037.0, "reward": 0.7834821790456772, "reward_std": 0.1881879549473524, "rewards/accuracy_reward/mean": 0.2946428544819355, "rewards/accuracy_reward/std": 0.45407117158174515, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05146361608058214, "step": 2005 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 858.9062805175781, "completions/mean_terminated_length": 758.0720062255859, "completions/min_length": 319.5, "completions/min_terminated_length": 319.5, "epoch": 0.5992084235680681, "grad_norm": 0.2999306917190552, "kl": 2.931640625, "learning_rate": 7.359165225313019e-06, "loss": 0.1557, "num_tokens": 980195379.0, "reward": 0.7059152126312256, "reward_std": 0.20072748884558678, "rewards/accuracy_reward/mean": 0.22098214365541935, "rewards/accuracy_reward/std": 0.3964175507426262, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05686598177999258, "step": 2006 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44866071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 889.8326416015625, "completions/mean_terminated_length": 784.0945892333984, "completions/min_length": 501.75, "completions/min_terminated_length": 501.75, "epoch": 0.5995071316555896, "grad_norm": 0.5159537196159363, "kl": 2.3515625, "learning_rate": 7.348645737565919e-06, "loss": 0.1203, "num_tokens": 980664600.0, "reward": 0.577566996216774, "reward_std": 0.14697497338056564, "rewards/accuracy_reward/mean": 0.08928571362048388, "rewards/accuracy_reward/std": 0.2784740626811981, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052180747501552105, "step": 2007 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41517857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 862.5647888183594, "completions/mean_terminated_length": 752.5802917480469, "completions/min_length": 321.25, "completions/min_terminated_length": 321.25, "epoch": 0.5998058397431111, "grad_norm": 0.2631114423274994, "kl": 2.76171875, "learning_rate": 7.338129404696223e-06, "loss": 0.1621, "num_tokens": 981131285.0, "reward": 0.6930803805589676, "reward_std": 0.19201872311532497, "rewards/accuracy_reward/mean": 0.20535714365541935, "rewards/accuracy_reward/std": 0.3721298538148403, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 2008 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 881.3080902099609, "completions/mean_terminated_length": 757.2720642089844, "completions/min_length": 260.5, "completions/min_terminated_length": 260.5, "epoch": 0.6001045478306325, "grad_norm": 0.659074068069458, "kl": 3.4921875, "learning_rate": 7.327616239217432e-06, "loss": 0.164, "num_tokens": 981599215.0, "reward": 0.6149553805589676, "reward_std": 0.14226302318274975, "rewards/accuracy_reward/mean": 0.13169642817229033, "rewards/accuracy_reward/std": 0.317329041659832, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06234363839030266, "step": 2009 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 861.4487152099609, "completions/mean_terminated_length": 765.4564208984375, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "epoch": 0.600403255918154, "grad_norm": 0.39345231652259827, "kl": 2.1328125, "learning_rate": 7.3171062536392874e-06, "loss": 0.109, "num_tokens": 982052264.0, "reward": 0.631138414144516, "reward_std": 0.18176084384322166, "rewards/accuracy_reward/mean": 0.14062500186264515, "rewards/accuracy_reward/std": 0.32547085359692574, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574134543538094, "step": 2010 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 889.3504791259766, "completions/mean_terminated_length": 768.7913665771484, "completions/min_length": 395.5, "completions/min_terminated_length": 395.5, "epoch": 0.6007019640056754, "grad_norm": 0.2625519037246704, "kl": 2.681640625, "learning_rate": 7.306599460467741e-06, "loss": 0.1532, "num_tokens": 982523301.0, "reward": 0.6378348618745804, "reward_std": 0.193947808817029, "rewards/accuracy_reward/mean": 0.1517857159487903, "rewards/accuracy_reward/std": 0.33460287749767303, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05585796292871237, "step": 2011 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 881.6785888671875, "completions/mean_terminated_length": 774.2588958740234, "completions/min_length": 292.5, "completions/min_terminated_length": 292.5, "epoch": 0.601000672093197, "grad_norm": 0.37261655926704407, "kl": 2.53125, "learning_rate": 7.296095872204952e-06, "loss": 0.132, "num_tokens": 982988613.0, "reward": 0.6869420111179352, "reward_std": 0.20291432924568653, "rewards/accuracy_reward/mean": 0.2008928582072258, "rewards/accuracy_reward/std": 0.39359766989946365, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.056609878316521645, "step": 2012 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.5, "completions/mean_length": 827.6652221679688, "completions/mean_terminated_length": 720.3813171386719, "completions/min_length": 380.5, "completions/min_terminated_length": 380.5, "epoch": 0.6012993801807184, "grad_norm": 0.26483026146888733, "kl": 2.306640625, "learning_rate": 7.285595501349259e-06, "loss": 0.1431, "num_tokens": 983432623.0, "reward": 0.7137277126312256, "reward_std": 0.1959347128868103, "rewards/accuracy_reward/mean": 0.22321428102441132, "rewards/accuracy_reward/std": 0.36166553758084774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047917463816702366, "step": 2013 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 869.0647735595703, "completions/mean_terminated_length": 744.2488555908203, "completions/min_length": 391.25, "completions/min_terminated_length": 391.25, "epoch": 0.6015980882682399, "grad_norm": 0.1856522262096405, "kl": 2.54296875, "learning_rate": 7.275098360395179e-06, "loss": 0.1359, "num_tokens": 983893484.0, "reward": 0.6210937798023224, "reward_std": 0.14281007833778858, "rewards/accuracy_reward/mean": 0.13392856903374195, "rewards/accuracy_reward/std": 0.33316638320684433, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 2014 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37946428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 843.2835388183594, "completions/mean_terminated_length": 734.4853515625, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "epoch": 0.6018967963557613, "grad_norm": 0.29811328649520874, "kl": 2.62109375, "learning_rate": 7.26460446183338e-06, "loss": 0.1499, "num_tokens": 984336571.0, "reward": 0.6752232387661934, "reward_std": 0.17252518609166145, "rewards/accuracy_reward/mean": 0.1934523843228817, "rewards/accuracy_reward/std": 0.3245402127504349, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 2015 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 849.0714569091797, "completions/mean_terminated_length": 729.7114410400391, "completions/min_length": 353.25, "completions/min_terminated_length": 353.25, "epoch": 0.6021955044432828, "grad_norm": 0.35825908184051514, "kl": 2.0810546875, "learning_rate": 7.25411381815068e-06, "loss": 0.1003, "num_tokens": 984794507.0, "reward": 0.686941996216774, "reward_std": 0.16501750517636538, "rewards/accuracy_reward/mean": 0.1964285708963871, "rewards/accuracy_reward/std": 0.3955099284648895, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.038242805283516645, "step": 2016 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33705357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 820.4687957763672, "completions/mean_terminated_length": 721.8253784179688, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "epoch": 0.6024942125308043, "grad_norm": 0.5900571346282959, "kl": 2.232421875, "learning_rate": 7.243626441830009e-06, "loss": 0.125, "num_tokens": 985235661.0, "reward": 0.6506696939468384, "reward_std": 0.13410672917962074, "rewards/accuracy_reward/mean": 0.16294642351567745, "rewards/accuracy_reward/std": 0.35680727660655975, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05192205682396889, "step": 2017 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 871.9933624267578, "completions/mean_terminated_length": 766.3004302978516, "completions/min_length": 321.25, "completions/min_terminated_length": 321.25, "epoch": 0.6027929206183258, "grad_norm": 0.18841180205345154, "kl": 1.673828125, "learning_rate": 7.233142345350428e-06, "loss": 0.0986, "num_tokens": 985697130.0, "reward": 0.6880580484867096, "reward_std": 0.17220556177198887, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.38421381264925003, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043343435507267714, "step": 2018 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 872.2232513427734, "completions/mean_terminated_length": 753.6419067382812, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "epoch": 0.6030916287058472, "grad_norm": 0.30982229113578796, "kl": 2.619140625, "learning_rate": 7.2226615411870796e-06, "loss": 0.1455, "num_tokens": 986160190.0, "reward": 0.663504496216774, "reward_std": 0.18723243102431297, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.3733755871653557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05320013873279095, "step": 2019 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 871.7678985595703, "completions/mean_terminated_length": 756.9355926513672, "completions/min_length": 409.75, "completions/min_terminated_length": 409.75, "epoch": 0.6033903367933687, "grad_norm": 0.28658556938171387, "kl": 2.630859375, "learning_rate": 7.212184041811198e-06, "loss": 0.1366, "num_tokens": 986625414.0, "reward": 0.638950914144516, "reward_std": 0.16378252767026424, "rewards/accuracy_reward/mean": 0.15178571734577417, "rewards/accuracy_reward/std": 0.326382651925087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05382578354328871, "step": 2020 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.25, "completions/mean_length": 793.1629791259766, "completions/mean_terminated_length": 693.9719696044922, "completions/min_length": 239.5, "completions/min_terminated_length": 239.5, "epoch": 0.6036890448808901, "grad_norm": 0.4062547981739044, "kl": 2.396484375, "learning_rate": 7.201709859690081e-06, "loss": 0.1415, "num_tokens": 987050383.0, "reward": 0.6417410969734192, "reward_std": 0.10314064472913742, "rewards/accuracy_reward/mean": 0.15364583488553762, "rewards/accuracy_reward/std": 0.32636887952685356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 2021 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 835.6674346923828, "completions/mean_terminated_length": 750.2105407714844, "completions/min_length": 396.75, "completions/min_terminated_length": 396.75, "epoch": 0.6039877529684117, "grad_norm": 0.35600870847702026, "kl": 2.33984375, "learning_rate": 7.191239007287082e-06, "loss": 0.1237, "num_tokens": 987490826.0, "reward": 0.664620578289032, "reward_std": 0.14675054140388966, "rewards/accuracy_reward/mean": 0.17559523787349463, "rewards/accuracy_reward/std": 0.3581012971699238, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 2022 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 818.2433166503906, "completions/mean_terminated_length": 730.1372222900391, "completions/min_length": 213.0, "completions/min_terminated_length": 213.0, "epoch": 0.6042864610559331, "grad_norm": 0.5395879745483398, "kl": 2.3515625, "learning_rate": 7.180771497061587e-06, "loss": 0.0982, "num_tokens": 987930871.0, "reward": 0.6372768133878708, "reward_std": 0.14184948056936264, "rewards/accuracy_reward/mean": 0.14732143189758062, "rewards/accuracy_reward/std": 0.3293979689478874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 2023 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 834.9910888671875, "completions/mean_terminated_length": 750.3454895019531, "completions/min_length": 394.5, "completions/min_terminated_length": 394.5, "epoch": 0.6045851691434545, "grad_norm": 0.5589957237243652, "kl": 2.6171875, "learning_rate": 7.1703073414690115e-06, "loss": 0.1387, "num_tokens": 988378771.0, "reward": 0.604910746216774, "reward_std": 0.15817278623580933, "rewards/accuracy_reward/mean": 0.11607143119908869, "rewards/accuracy_reward/std": 0.27471249364316463, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05157411843538284, "step": 2024 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 841.4553833007812, "completions/mean_terminated_length": 767.9867553710938, "completions/min_length": 307.75, "completions/min_terminated_length": 307.75, "epoch": 0.604883877230976, "grad_norm": 0.531307578086853, "kl": 2.4609375, "learning_rate": 7.159846552960774e-06, "loss": 0.1346, "num_tokens": 988827743.0, "reward": 0.6640625298023224, "reward_std": 0.23897065594792366, "rewards/accuracy_reward/mean": 0.1741071455180645, "rewards/accuracy_reward/std": 0.3758445903658867, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04903263598680496, "step": 2025 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 833.3147735595703, "completions/mean_terminated_length": 734.2561340332031, "completions/min_length": 276.5, "completions/min_terminated_length": 276.5, "epoch": 0.6051825853184974, "grad_norm": 0.2091558575630188, "kl": 2.283203125, "learning_rate": 7.149389143984295e-06, "loss": 0.1322, "num_tokens": 989275004.0, "reward": 0.6289062798023224, "reward_std": 0.14120302209630609, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.2849196195602417, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 2026 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 827.2232513427734, "completions/mean_terminated_length": 740.7761993408203, "completions/min_length": 337.25, "completions/min_terminated_length": 337.25, "epoch": 0.605481293406019, "grad_norm": 0.3009856343269348, "kl": 2.193359375, "learning_rate": 7.13893512698296e-06, "loss": 0.1234, "num_tokens": 989720784.0, "reward": 0.6077009290456772, "reward_std": 0.10415794141590595, "rewards/accuracy_reward/mean": 0.11830357275903225, "rewards/accuracy_reward/std": 0.27232028916478157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 2027 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2633928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 801.3326263427734, "completions/mean_terminated_length": 724.3906097412109, "completions/min_length": 419.25, "completions/min_terminated_length": 419.25, "epoch": 0.6057800014935404, "grad_norm": 0.22933541238307953, "kl": 2.619140625, "learning_rate": 7.12848451439613e-06, "loss": 0.1511, "num_tokens": 990153365.0, "reward": 0.6160714477300644, "reward_std": 0.14133240096271038, "rewards/accuracy_reward/mean": 0.13913690764456987, "rewards/accuracy_reward/std": 0.31442492455244064, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392761349678, "rewards/tag_count_reward/std": 0.050723335705697536, "step": 2028 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 801.7500457763672, "completions/mean_terminated_length": 735.0033874511719, "completions/min_length": 357.75, "completions/min_terminated_length": 357.75, "epoch": 0.6060787095810619, "grad_norm": 0.20068176090717316, "kl": 1.724609375, "learning_rate": 7.118037318659108e-06, "loss": 0.0954, "num_tokens": 990585701.0, "reward": 0.6383928954601288, "reward_std": 0.13895082846283913, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.35326216369867325, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03934345254674554, "step": 2029 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27008928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 781.9174499511719, "completions/mean_terminated_length": 694.1792297363281, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "epoch": 0.6063774176685833, "grad_norm": 0.3276822566986084, "kl": 2.267578125, "learning_rate": 7.10759355220314e-06, "loss": 0.1284, "num_tokens": 991006016.0, "reward": 0.667410746216774, "reward_std": 0.13552370108664036, "rewards/accuracy_reward/mean": 0.17633928917348385, "rewards/accuracy_reward/std": 0.3696085959672928, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04479066748172045, "step": 2030 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.24553571428571427, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 782.4397583007812, "completions/mean_terminated_length": 707.8205718994141, "completions/min_length": 408.25, "completions/min_terminated_length": 408.25, "epoch": 0.6066761257561049, "grad_norm": 0.3900409936904907, "kl": 2.24609375, "learning_rate": 7.097153227455379e-06, "loss": 0.1298, "num_tokens": 991426517.0, "reward": 0.6992187798023224, "reward_std": 0.1551669016480446, "rewards/accuracy_reward/mean": 0.2217261902987957, "rewards/accuracy_reward/std": 0.40160931646823883, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.040945359505712986, "step": 2031 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 825.1964721679688, "completions/mean_terminated_length": 734.8288116455078, "completions/min_length": 357.25, "completions/min_terminated_length": 357.25, "epoch": 0.6069748338436263, "grad_norm": 0.3397832214832306, "kl": 2.296875, "learning_rate": 7.0867163568388895e-06, "loss": 0.1172, "num_tokens": 991868909.0, "reward": 0.6411830484867096, "reward_std": 0.13822929561138153, "rewards/accuracy_reward/mean": 0.14955357275903225, "rewards/accuracy_reward/std": 0.3527960479259491, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 2032 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33705357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 871.7902069091797, "completions/mean_terminated_length": 796.2234191894531, "completions/min_length": 416.75, "completions/min_terminated_length": 416.75, "epoch": 0.6072735419311478, "grad_norm": 0.3231644034385681, "kl": 2.2421875, "learning_rate": 7.076282952772634e-06, "loss": 0.1045, "num_tokens": 992329071.0, "reward": 0.659598246216774, "reward_std": 0.1642799824476242, "rewards/accuracy_reward/mean": 0.1718749995343387, "rewards/accuracy_reward/std": 0.31353943422436714, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 2033 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 903.747802734375, "completions/mean_terminated_length": 806.8131408691406, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "epoch": 0.6075722500186692, "grad_norm": 0.2839789390563965, "kl": 1.888671875, "learning_rate": 7.065853027671433e-06, "loss": 0.0994, "num_tokens": 992804846.0, "reward": 0.6841518133878708, "reward_std": 0.16974328085780144, "rewards/accuracy_reward/mean": 0.19419643096625805, "rewards/accuracy_reward/std": 0.38859422504901886, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 2034 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.26339285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 810.0736999511719, "completions/mean_terminated_length": 735.8305816650391, "completions/min_length": 309.25, "completions/min_terminated_length": 309.25, "epoch": 0.6078709581061907, "grad_norm": 0.2261303812265396, "kl": 2.1494140625, "learning_rate": 7.055426593945982e-06, "loss": 0.1188, "num_tokens": 993248751.0, "reward": 0.5998884290456772, "reward_std": 0.15902957506477833, "rewards/accuracy_reward/mean": 0.10937500232830644, "rewards/accuracy_reward/std": 0.2894482687115669, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.0458291289396584, "step": 2035 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 855.6875457763672, "completions/mean_terminated_length": 751.880859375, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "epoch": 0.6081696661937122, "grad_norm": 0.46518635749816895, "kl": 2.177734375, "learning_rate": 7.04500366400281e-06, "loss": 0.1151, "num_tokens": 993698803.0, "reward": 0.5636160969734192, "reward_std": 0.11848736554384232, "rewards/accuracy_reward/mean": 0.07589285611175, "rewards/accuracy_reward/std": 0.2249168772250414, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.053750067949295044, "step": 2036 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38169642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 843.513427734375, "completions/mean_terminated_length": 734.2190704345703, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.6084683742812337, "grad_norm": 0.215770423412323, "kl": 2.087890625, "learning_rate": 7.034584250244292e-06, "loss": 0.1051, "num_tokens": 994145881.0, "reward": 0.6696428954601288, "reward_std": 0.2237712573260069, "rewards/accuracy_reward/mean": 0.18080357369035482, "rewards/accuracy_reward/std": 0.3499346375465393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051574116572737694, "step": 2037 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 881.6116333007812, "completions/mean_terminated_length": 763.6698913574219, "completions/min_length": 393.5, "completions/min_terminated_length": 393.5, "epoch": 0.6087670823687551, "grad_norm": 0.3789597451686859, "kl": 1.81640625, "learning_rate": 7.024168365068601e-06, "loss": 0.0982, "num_tokens": 994614731.0, "reward": 0.6858259290456772, "reward_std": 0.16604185104370117, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.3922537863254547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2038 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 841.404052734375, "completions/mean_terminated_length": 718.8861541748047, "completions/min_length": 343.5, "completions/min_terminated_length": 343.5, "epoch": 0.6090657904562766, "grad_norm": 0.2406461238861084, "kl": 2.29296875, "learning_rate": 7.0137560208697264e-06, "loss": 0.1149, "num_tokens": 995070736.0, "reward": 0.567522332072258, "reward_std": 0.12708964571356773, "rewards/accuracy_reward/mean": 0.08035714225843549, "rewards/accuracy_reward/std": 0.2557261884212494, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054577698931097984, "step": 2039 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 857.1897583007812, "completions/mean_terminated_length": 756.1214599609375, "completions/min_length": 421.25, "completions/min_terminated_length": 421.25, "epoch": 0.609364498543798, "grad_norm": 0.5025520324707031, "kl": 2.08203125, "learning_rate": 7.003347230037434e-06, "loss": 0.115, "num_tokens": 995529381.0, "reward": 0.6456473544239998, "reward_std": 0.12709948047995567, "rewards/accuracy_reward/mean": 0.1562500037252903, "rewards/accuracy_reward/std": 0.2912540137767792, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 2040 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 858.7567443847656, "completions/mean_terminated_length": 758.6382904052734, "completions/min_length": 377.0, "completions/min_terminated_length": 377.0, "epoch": 0.6096632066313196, "grad_norm": 0.21443721652030945, "kl": 2.69140625, "learning_rate": 6.992942004957271e-06, "loss": 0.1405, "num_tokens": 995988680.0, "reward": 0.5424107387661934, "reward_std": 0.08111475128680468, "rewards/accuracy_reward/mean": 0.05580357299186289, "rewards/accuracy_reward/std": 0.16640411131083965, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.056180731393396854, "step": 2041 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 869.2611999511719, "completions/mean_terminated_length": 741.53857421875, "completions/min_length": 363.5, "completions/min_terminated_length": 363.5, "epoch": 0.609961914718841, "grad_norm": 0.3495849668979645, "kl": 3.005859375, "learning_rate": 6.98254035801053e-06, "loss": 0.1551, "num_tokens": 996451357.0, "reward": 0.6707589477300644, "reward_std": 0.1888755802065134, "rewards/accuracy_reward/mean": 0.1852678544819355, "rewards/accuracy_reward/std": 0.380618192255497, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05737193766981363, "step": 2042 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 844.0558471679688, "completions/mean_terminated_length": 727.4985809326172, "completions/min_length": 274.5, "completions/min_terminated_length": 274.5, "epoch": 0.6102606228063625, "grad_norm": 0.28839316964149475, "kl": 2.51953125, "learning_rate": 6.972142301574256e-06, "loss": 0.1215, "num_tokens": 996908454.0, "reward": 0.7393973469734192, "reward_std": 0.18953024595975876, "rewards/accuracy_reward/mean": 0.2499999962747097, "rewards/accuracy_reward/std": 0.4105956479907036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403157249093056, "step": 2043 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 917.5201263427734, "completions/mean_terminated_length": 808.3970031738281, "completions/min_length": 477.25, "completions/min_terminated_length": 477.25, "epoch": 0.6105593308938839, "grad_norm": 0.2854621708393097, "kl": 3.5703125, "learning_rate": 6.9617478480212145e-06, "loss": 0.1767, "num_tokens": 997391423.0, "reward": 0.605468787252903, "reward_std": 0.102937295101583, "rewards/accuracy_reward/mean": 0.1320684514939785, "rewards/accuracy_reward/std": 0.28121770918369293, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06277689151465893, "step": 2044 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 875.0491485595703, "completions/mean_terminated_length": 777.0268249511719, "completions/min_length": 419.0, "completions/min_terminated_length": 419.0, "epoch": 0.6108580389814054, "grad_norm": 0.3648027181625366, "kl": 3.169921875, "learning_rate": 6.951357009719893e-06, "loss": 0.1668, "num_tokens": 997859269.0, "reward": 0.6618303954601288, "reward_std": 0.17558006197214127, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.3716098964214325, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.057596758008003235, "step": 2045 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 878.2902069091797, "completions/mean_terminated_length": 742.7449951171875, "completions/min_length": 384.75, "completions/min_terminated_length": 384.75, "epoch": 0.6111567470689269, "grad_norm": 0.35412999987602234, "kl": 2.451171875, "learning_rate": 6.940969799034465e-06, "loss": 0.1201, "num_tokens": 998326391.0, "reward": 0.6713169813156128, "reward_std": 0.1590778212994337, "rewards/accuracy_reward/mean": 0.18303571455180645, "rewards/accuracy_reward/std": 0.375056616961956, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918293237686, "step": 2046 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 858.8348541259766, "completions/mean_terminated_length": 756.8016662597656, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "epoch": 0.6114554551564484, "grad_norm": 0.23921418190002441, "kl": 2.806640625, "learning_rate": 6.9305862283248005e-06, "loss": 0.1509, "num_tokens": 998778493.0, "reward": 0.6110491305589676, "reward_std": 0.13793670013546944, "rewards/accuracy_reward/mean": 0.12276785774156451, "rewards/accuracy_reward/std": 0.3050954341888428, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05178379639983177, "step": 2047 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 845.1138763427734, "completions/mean_terminated_length": 732.2964630126953, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "epoch": 0.6117541632439698, "grad_norm": 0.282151460647583, "kl": 2.4453125, "learning_rate": 6.920206309946425e-06, "loss": 0.1289, "num_tokens": 999223680.0, "reward": 0.690848246216774, "reward_std": 0.1857914887368679, "rewards/accuracy_reward/mean": 0.20089286100119352, "rewards/accuracy_reward/std": 0.37768761068582535, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 2048 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 904.7969360351562, "completions/mean_terminated_length": 792.2074432373047, "completions/min_length": 360.25, "completions/min_terminated_length": 360.25, "epoch": 0.6120528713314913, "grad_norm": 0.4849480390548706, "kl": 3.20703125, "learning_rate": 6.909830056250527e-06, "loss": 0.1505, "num_tokens": 999696741.0, "reward": 0.5954241156578064, "reward_std": 0.13991297595202923, "rewards/accuracy_reward/mean": 0.11160714318975806, "rewards/accuracy_reward/std": 0.30108752846717834, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.0571157643571496, "step": 2049 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39732142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 847.9152221679688, "completions/mean_terminated_length": 736.5578918457031, "completions/min_length": 371.5, "completions/min_terminated_length": 371.5, "epoch": 0.6123515794190127, "grad_norm": 0.22639167308807373, "kl": 2.9921875, "learning_rate": 6.899457479583931e-06, "loss": 0.1468, "num_tokens": 1000158127.0, "reward": 0.6216517984867096, "reward_std": 0.10877960966899991, "rewards/accuracy_reward/mean": 0.1361607185099274, "rewards/accuracy_reward/std": 0.28671410493552685, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05698250140994787, "step": 2050 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 837.2991485595703, "completions/mean_terminated_length": 736.4647064208984, "completions/min_length": 339.25, "completions/min_terminated_length": 339.25, "epoch": 0.6126502875065343, "grad_norm": 0.19986121356487274, "kl": 2.072265625, "learning_rate": 6.889088592289092e-06, "loss": 0.1091, "num_tokens": 1000615349.0, "reward": 0.7684152126312256, "reward_std": 0.1696819607168436, "rewards/accuracy_reward/mean": 0.2767857126891613, "rewards/accuracy_reward/std": 0.43795011192560196, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098951727152, "step": 2051 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 885.7165679931641, "completions/mean_terminated_length": 771.2224731445312, "completions/min_length": 391.25, "completions/min_terminated_length": 391.25, "epoch": 0.6129489955940557, "grad_norm": 0.38223588466644287, "kl": 3.81640625, "learning_rate": 6.878723406704064e-06, "loss": 0.2009, "num_tokens": 1001088550.0, "reward": 0.646763414144516, "reward_std": 0.1885322667658329, "rewards/accuracy_reward/mean": 0.16741071501746774, "rewards/accuracy_reward/std": 0.3368151895701885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526828289032, "rewards/tag_count_reward/std": 0.06882160902023315, "step": 2052 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 850.7299652099609, "completions/mean_terminated_length": 744.2432861328125, "completions/min_length": 343.75, "completions/min_terminated_length": 343.75, "epoch": 0.6132477036815772, "grad_norm": 0.22607791423797607, "kl": 2.27734375, "learning_rate": 6.8683619351625065e-06, "loss": 0.1077, "num_tokens": 1001546253.0, "reward": 0.5703125149011612, "reward_std": 0.1504520196467638, "rewards/accuracy_reward/mean": 0.08035714388825, "rewards/accuracy_reward/std": 0.2446136437356472, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2053 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 835.8281707763672, "completions/mean_terminated_length": 746.0986175537109, "completions/min_length": 284.25, "completions/min_terminated_length": 284.25, "epoch": 0.6135464117690986, "grad_norm": 0.2140842229127884, "kl": 2.541015625, "learning_rate": 6.85800418999365e-06, "loss": 0.1218, "num_tokens": 1001993424.0, "reward": 0.577566996216774, "reward_std": 0.119900643825531, "rewards/accuracy_reward/mean": 0.09151785890571773, "rewards/accuracy_reward/std": 0.2500599529594183, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.0528625980950892, "step": 2054 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 876.8125305175781, "completions/mean_terminated_length": 770.0, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.6138451198566202, "grad_norm": 0.19538654386997223, "kl": 2.634765625, "learning_rate": 6.847650183522307e-06, "loss": 0.1314, "num_tokens": 1002470124.0, "reward": 0.5636160969734192, "reward_std": 0.14752530120313168, "rewards/accuracy_reward/mean": 0.07589285727590322, "rewards/accuracy_reward/std": 0.2587081752717495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053152467124164104, "step": 2055 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 870.638427734375, "completions/mean_terminated_length": 759.5768127441406, "completions/min_length": 282.25, "completions/min_terminated_length": 282.25, "epoch": 0.6141438279441416, "grad_norm": 0.27847978472709656, "kl": 2.3359375, "learning_rate": 6.8372999280688175e-06, "loss": 0.1003, "num_tokens": 1002933738.0, "reward": 0.564732164144516, "reward_std": 0.1602490758523345, "rewards/accuracy_reward/mean": 0.07589285937137902, "rewards/accuracy_reward/std": 0.23633724264800549, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04931834805756807, "step": 2056 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 833.8951416015625, "completions/mean_terminated_length": 731.9572906494141, "completions/min_length": 328.75, "completions/min_terminated_length": 328.75, "epoch": 0.6144425360316631, "grad_norm": 0.3537287712097168, "kl": 2.5859375, "learning_rate": 6.826953435949081e-06, "loss": 0.1318, "num_tokens": 1003382955.0, "reward": 0.6339286118745804, "reward_std": 0.11999470554292202, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.34222330898046494, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04620361328125, "step": 2057 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 838.5089721679688, "completions/mean_terminated_length": 749.5872497558594, "completions/min_length": 365.5, "completions/min_terminated_length": 365.5, "epoch": 0.6147412441191845, "grad_norm": 0.19874462485313416, "kl": 1.5634765625, "learning_rate": 6.816610719474503e-06, "loss": 0.0799, "num_tokens": 1003826239.0, "reward": 0.6847098469734192, "reward_std": 0.15622897259891033, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.3724236823618412, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.0412445142865181, "step": 2058 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2455357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 753.0312805175781, "completions/mean_terminated_length": 666.4080810546875, "completions/min_length": 165.25, "completions/min_terminated_length": 165.25, "epoch": 0.615039952206706, "grad_norm": 0.3760141432285309, "kl": 1.54296875, "learning_rate": 6.806271790952008e-06, "loss": 0.1054, "num_tokens": 1004234461.0, "reward": 0.8169643133878708, "reward_std": 0.2569829672574997, "rewards/accuracy_reward/mean": 0.3236607201397419, "rewards/accuracy_reward/std": 0.4434858486056328, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2059 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2901785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 803.7522735595703, "completions/mean_terminated_length": 713.6941528320312, "completions/min_length": 301.5, "completions/min_terminated_length": 301.5, "epoch": 0.6153386602942275, "grad_norm": 0.2327592521905899, "kl": 2.0048828125, "learning_rate": 6.795936662684004e-06, "loss": 0.1017, "num_tokens": 1004659630.0, "reward": 0.674107164144516, "reward_std": 0.11611604923382401, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.37671683728694916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044658167753368616, "step": 2060 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 853.7478179931641, "completions/mean_terminated_length": 739.3461456298828, "completions/min_length": 267.5, "completions/min_terminated_length": 267.5, "epoch": 0.615637368381749, "grad_norm": 0.21979083120822906, "kl": 2.283203125, "learning_rate": 6.785605346968387e-06, "loss": 0.1161, "num_tokens": 1005113133.0, "reward": 0.6250000298023224, "reward_std": 0.15260480530560017, "rewards/accuracy_reward/mean": 0.13839285378344357, "rewards/accuracy_reward/std": 0.3004226069897413, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.055638475343585014, "step": 2061 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 841.1964569091797, "completions/mean_terminated_length": 750.0451812744141, "completions/min_length": 286.75, "completions/min_terminated_length": 286.75, "epoch": 0.6159360764692704, "grad_norm": 0.32368335127830505, "kl": 2.30859375, "learning_rate": 6.775277856098501e-06, "loss": 0.1321, "num_tokens": 1005566693.0, "reward": 0.6445312798023224, "reward_std": 0.16574986837804317, "rewards/accuracy_reward/mean": 0.15625000186264515, "rewards/accuracy_reward/std": 0.3321193791925907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05134009011089802, "step": 2062 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28348214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 815.9308319091797, "completions/mean_terminated_length": 733.7039337158203, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "epoch": 0.6162347845567919, "grad_norm": 0.43043047189712524, "kl": 2.400390625, "learning_rate": 6.7649542023631545e-06, "loss": 0.1523, "num_tokens": 1006000710.0, "reward": 0.7209821790456772, "reward_std": 0.22036054357886314, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.41468068957328796, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05581916682422161, "step": 2063 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 796.7857513427734, "completions/mean_terminated_length": 711.5524139404297, "completions/min_length": 349.25, "completions/min_terminated_length": 349.25, "epoch": 0.6165334926443133, "grad_norm": 0.4371796250343323, "kl": 2.1796875, "learning_rate": 6.754634398046578e-06, "loss": 0.1328, "num_tokens": 1006433734.0, "reward": 0.612723246216774, "reward_std": 0.13681860826909542, "rewards/accuracy_reward/mean": 0.12276785913854837, "rewards/accuracy_reward/std": 0.30685120448470116, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 2064 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 813.6897583007812, "completions/mean_terminated_length": 706.7332458496094, "completions/min_length": 190.0, "completions/min_terminated_length": 190.0, "epoch": 0.6168322007318349, "grad_norm": 0.42221564054489136, "kl": 2.65625, "learning_rate": 6.744318455428436e-06, "loss": 0.157, "num_tokens": 1006876811.0, "reward": 0.668526828289032, "reward_std": 0.19726279377937317, "rewards/accuracy_reward/mean": 0.18303571548312902, "rewards/accuracy_reward/std": 0.37157008051872253, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05613203626126051, "step": 2065 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 868.9152221679688, "completions/mean_terminated_length": 750.9327697753906, "completions/min_length": 242.25, "completions/min_terminated_length": 242.25, "epoch": 0.6171309088193563, "grad_norm": 0.21905602514743805, "kl": 2.3837890625, "learning_rate": 6.73400638678378e-06, "loss": 0.1222, "num_tokens": 1007341861.0, "reward": 0.594866082072258, "reward_std": 0.18585958145558834, "rewards/accuracy_reward/mean": 0.10714285774156451, "rewards/accuracy_reward/std": 0.28742675110697746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05140746245160699, "step": 2066 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 801.0379791259766, "completions/mean_terminated_length": 718.8246459960938, "completions/min_length": 254.25, "completions/min_terminated_length": 254.25, "epoch": 0.6174296169068777, "grad_norm": 0.2811179459095001, "kl": 2.302734375, "learning_rate": 6.723698204383067e-06, "loss": 0.1282, "num_tokens": 1007775350.0, "reward": 0.7159598469734192, "reward_std": 0.16830307990312576, "rewards/accuracy_reward/mean": 0.2254464253783226, "rewards/accuracy_reward/std": 0.40472397953271866, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 2067 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 848.5424499511719, "completions/mean_terminated_length": 716.3337707519531, "completions/min_length": 237.5, "completions/min_terminated_length": 237.5, "epoch": 0.6177283249943992, "grad_norm": 0.22578053176403046, "kl": 2.203125, "learning_rate": 6.713393920492119e-06, "loss": 0.1041, "num_tokens": 1008228537.0, "reward": 0.6004464477300644, "reward_std": 0.14198577962815762, "rewards/accuracy_reward/mean": 0.10937499860301614, "rewards/accuracy_reward/std": 0.23348690196871758, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04605984315276146, "step": 2068 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.171875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 728.1986846923828, "completions/mean_terminated_length": 667.3649139404297, "completions/min_length": 182.25, "completions/min_terminated_length": 182.25, "epoch": 0.6180270330819206, "grad_norm": 0.4317675828933716, "kl": 2.20703125, "learning_rate": 6.70309354737213e-06, "loss": 0.1327, "num_tokens": 1008623458.0, "reward": 0.7354910969734192, "reward_std": 0.11500824987888336, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.3776872605085373, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0410674219019711, "step": 2069 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 816.6897735595703, "completions/mean_terminated_length": 721.7075958251953, "completions/min_length": 207.75, "completions/min_terminated_length": 207.75, "epoch": 0.6183257411694422, "grad_norm": 0.5248735547065735, "kl": 4.205078125, "learning_rate": 6.6927970972796255e-06, "loss": 0.2105, "num_tokens": 1009059207.0, "reward": 0.655691996216774, "reward_std": 0.16579223796725273, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.34726569056510925, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.05893508065491915, "step": 2070 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 805.3683471679688, "completions/mean_terminated_length": 719.1063079833984, "completions/min_length": 203.75, "completions/min_terminated_length": 203.75, "epoch": 0.6186244492569636, "grad_norm": 0.358720600605011, "kl": 2.56640625, "learning_rate": 6.682504582466482e-06, "loss": 0.1261, "num_tokens": 1009488652.0, "reward": 0.5842634215950966, "reward_std": 0.09840369643643498, "rewards/accuracy_reward/mean": 0.09375, "rewards/accuracy_reward/std": 0.19594908505678177, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 2071 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 799.0692291259766, "completions/mean_terminated_length": 695.0828704833984, "completions/min_length": 262.25, "completions/min_terminated_length": 262.25, "epoch": 0.6189231573444851, "grad_norm": 0.5766188502311707, "kl": 3.6875, "learning_rate": 6.672216015179873e-06, "loss": 0.2025, "num_tokens": 1009915387.0, "reward": 0.6082589626312256, "reward_std": 0.13709110021591187, "rewards/accuracy_reward/mean": 0.12053571082651615, "rewards/accuracy_reward/std": 0.30219658464193344, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 2072 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2901785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 814.5558471679688, "completions/mean_terminated_length": 730.8179321289062, "completions/min_length": 158.5, "completions/min_terminated_length": 158.5, "epoch": 0.6192218654320065, "grad_norm": 0.3898375332355499, "kl": 2.755859375, "learning_rate": 6.661931407662292e-06, "loss": 0.1616, "num_tokens": 1010353588.0, "reward": 0.615513414144516, "reward_std": 0.14394082874059677, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.3305374011397362, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04571862844750285, "step": 2073 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 852.7388763427734, "completions/mean_terminated_length": 749.5203704833984, "completions/min_length": 271.5, "completions/min_terminated_length": 271.5, "epoch": 0.619520573519528, "grad_norm": 0.15633521974086761, "kl": 1.69140625, "learning_rate": 6.65165077215151e-06, "loss": 0.0799, "num_tokens": 1010808031.0, "reward": 0.593191996216774, "reward_std": 0.12972580455243587, "rewards/accuracy_reward/mean": 0.10044643143191934, "rewards/accuracy_reward/std": 0.2811446264386177, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2074 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 789.1272583007812, "completions/mean_terminated_length": 698.805908203125, "completions/min_length": 244.25, "completions/min_terminated_length": 244.25, "epoch": 0.6198192816070495, "grad_norm": 0.22504667937755585, "kl": 2.322265625, "learning_rate": 6.6413741208805795e-06, "loss": 0.1424, "num_tokens": 1011231560.0, "reward": 0.6886161118745804, "reward_std": 0.21809037402272224, "rewards/accuracy_reward/mean": 0.1986607126891613, "rewards/accuracy_reward/std": 0.3657253868877888, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 2075 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23214285714285712, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 786.3616485595703, "completions/mean_terminated_length": 717.1237640380859, "completions/min_length": 317.0, "completions/min_terminated_length": 317.0, "epoch": 0.620117989694571, "grad_norm": 0.45390623807907104, "kl": 2.353515625, "learning_rate": 6.631101466077801e-06, "loss": 0.134, "num_tokens": 1011660074.0, "reward": 0.6975446790456772, "reward_std": 0.2121532689779997, "rewards/accuracy_reward/mean": 0.2053571380674839, "rewards/accuracy_reward/std": 0.4012810215353966, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.042415473610162735, "step": 2076 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 821.3795013427734, "completions/mean_terminated_length": 726.6328582763672, "completions/min_length": 191.0, "completions/min_terminated_length": 191.0, "epoch": 0.6204166977820924, "grad_norm": 0.23327866196632385, "kl": 2.212890625, "learning_rate": 6.6208328199667305e-06, "loss": 0.1228, "num_tokens": 1012105012.0, "reward": 0.6143973469734192, "reward_std": 0.16053726384416223, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.2741965651512146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 2077 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2544642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 785.8794860839844, "completions/mean_terminated_length": 703.4151611328125, "completions/min_length": 129.5, "completions/min_terminated_length": 129.5, "epoch": 0.6207154058696139, "grad_norm": 0.2113039195537567, "kl": 1.955078125, "learning_rate": 6.610568194766152e-06, "loss": 0.1125, "num_tokens": 1012537822.0, "reward": 0.6886160969734192, "reward_std": 0.1831626519560814, "rewards/accuracy_reward/mean": 0.1964285671710968, "rewards/accuracy_reward/std": 0.31617070734500885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 2078 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 852.232177734375, "completions/mean_terminated_length": 758.9851226806641, "completions/min_length": 204.75, "completions/min_terminated_length": 204.75, "epoch": 0.6210141139571353, "grad_norm": 0.250117689371109, "kl": 1.5859375, "learning_rate": 6.600307602690057e-06, "loss": 0.0884, "num_tokens": 1012995734.0, "reward": 0.6283482313156128, "reward_std": 0.15901843272149563, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.3233402781188488, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.03511275444179773, "step": 2079 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 786.2678985595703, "completions/mean_terminated_length": 686.190673828125, "completions/min_length": 291.75, "completions/min_terminated_length": 291.75, "epoch": 0.6213128220446569, "grad_norm": 0.36773473024368286, "kl": 1.72265625, "learning_rate": 6.590051055947653e-06, "loss": 0.1029, "num_tokens": 1013418382.0, "reward": 0.6746651977300644, "reward_std": 0.18806209415197372, "rewards/accuracy_reward/mean": 0.18303571082651615, "rewards/accuracy_reward/std": 0.37314169853925705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 2080 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 832.6987152099609, "completions/mean_terminated_length": 742.1912078857422, "completions/min_length": 190.75, "completions/min_terminated_length": 190.75, "epoch": 0.6216115301321783, "grad_norm": 0.34770703315734863, "kl": 1.4970703125, "learning_rate": 6.579798566743314e-06, "loss": 0.0873, "num_tokens": 1013871271.0, "reward": 0.610491082072258, "reward_std": 0.16510027274489403, "rewards/accuracy_reward/mean": 0.11830357369035482, "rewards/accuracy_reward/std": 0.28768257051706314, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041067422833293676, "step": 2081 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 871.5848541259766, "completions/mean_terminated_length": 749.5315399169922, "completions/min_length": 157.75, "completions/min_terminated_length": 157.75, "epoch": 0.6219102382196998, "grad_norm": 0.2431933879852295, "kl": 1.529296875, "learning_rate": 6.569550147276603e-06, "loss": 0.0737, "num_tokens": 1014337149.0, "reward": 0.6149553805589676, "reward_std": 0.12421061471104622, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.32806215435266495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043574148789048195, "step": 2082 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 819.4241485595703, "completions/mean_terminated_length": 716.4209594726562, "completions/min_length": 171.0, "completions/min_terminated_length": 171.0, "epoch": 0.6222089463072212, "grad_norm": 0.2467096596956253, "kl": 1.740234375, "learning_rate": 6.5593058097422315e-06, "loss": 0.0946, "num_tokens": 1014779531.0, "reward": 0.658482164144516, "reward_std": 0.15492141246795654, "rewards/accuracy_reward/mean": 0.16517857555299997, "rewards/accuracy_reward/std": 0.3358650878071785, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.037730947602540255, "step": 2083 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 827.8705749511719, "completions/mean_terminated_length": 715.3681030273438, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "epoch": 0.6225076543947428, "grad_norm": 0.23921717703342438, "kl": 1.4619140625, "learning_rate": 6.54906556633006e-06, "loss": 0.0858, "num_tokens": 1015224289.0, "reward": 0.6886160969734192, "reward_std": 0.20184666849672794, "rewards/accuracy_reward/mean": 0.1964285671710968, "rewards/accuracy_reward/std": 0.37484046816825867, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 2084 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 837.1294860839844, "completions/mean_terminated_length": 735.8040161132812, "completions/min_length": 387.75, "completions/min_terminated_length": 387.75, "epoch": 0.6228063624822642, "grad_norm": 0.24906381964683533, "kl": 1.1103515625, "learning_rate": 6.538829429225068e-06, "loss": 0.0569, "num_tokens": 1015668299.0, "reward": 0.6188616305589676, "reward_std": 0.13230968080461025, "rewards/accuracy_reward/mean": 0.12760416883975267, "rewards/accuracy_reward/std": 0.30797868967056274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 2085 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 891.1272735595703, "completions/mean_terminated_length": 779.7418975830078, "completions/min_length": 339.75, "completions/min_terminated_length": 339.75, "epoch": 0.6231050705697857, "grad_norm": 0.3551114499568939, "kl": 1.8125, "learning_rate": 6.528597410607364e-06, "loss": 0.0938, "num_tokens": 1016138436.0, "reward": 0.6763393133878708, "reward_std": 0.18559064902365208, "rewards/accuracy_reward/mean": 0.18526785937137902, "rewards/accuracy_reward/std": 0.3437926974147558, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04660273063927889, "step": 2086 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 876.4754791259766, "completions/mean_terminated_length": 772.517333984375, "completions/min_length": 336.25, "completions/min_terminated_length": 336.25, "epoch": 0.6234037786573071, "grad_norm": 0.3012208342552185, "kl": 1.205078125, "learning_rate": 6.518369522652136e-06, "loss": 0.0917, "num_tokens": 1016597481.0, "reward": 0.7645089775323868, "reward_std": 0.23889075219631195, "rewards/accuracy_reward/mean": 0.2700892873108387, "rewards/accuracy_reward/std": 0.41565291583538055, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.03659330680966377, "step": 2087 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 851.0870971679688, "completions/mean_terminated_length": 746.1733551025391, "completions/min_length": 302.75, "completions/min_terminated_length": 302.75, "epoch": 0.6237024867448286, "grad_norm": 0.2301018387079239, "kl": 1.99609375, "learning_rate": 6.508145777529673e-06, "loss": 0.1229, "num_tokens": 1017049888.0, "reward": 0.6004464626312256, "reward_std": 0.1558859832584858, "rewards/accuracy_reward/mean": 0.10937500046566129, "rewards/accuracy_reward/std": 0.2891690544784069, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589571990072727, "step": 2088 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 858.6094207763672, "completions/mean_terminated_length": 738.7389831542969, "completions/min_length": 300.25, "completions/min_terminated_length": 300.25, "epoch": 0.6240011948323501, "grad_norm": 0.30669382214546204, "kl": 1.69921875, "learning_rate": 6.497926187405326e-06, "loss": 0.0806, "num_tokens": 1017511217.0, "reward": 0.6707589626312256, "reward_std": 0.17142805643379688, "rewards/accuracy_reward/mean": 0.19159226049669087, "rewards/accuracy_reward/std": 0.337375370785594, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 2089 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 871.6741485595703, "completions/mean_terminated_length": 770.0023498535156, "completions/min_length": 207.0, "completions/min_terminated_length": 207.0, "epoch": 0.6242999029198716, "grad_norm": 0.3203933537006378, "kl": 1.5625, "learning_rate": 6.487710764439508e-06, "loss": 0.0799, "num_tokens": 1017970303.0, "reward": 0.598772332072258, "reward_std": 0.10843916796147823, "rewards/accuracy_reward/mean": 0.10714285727590322, "rewards/accuracy_reward/std": 0.30529554188251495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2090 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3147321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 813.763427734375, "completions/mean_terminated_length": 714.5162963867188, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "epoch": 0.624598611007393, "grad_norm": 0.19501993060112, "kl": 1.501953125, "learning_rate": 6.4774995207876654e-06, "loss": 0.1001, "num_tokens": 1018410789.0, "reward": 0.6250000298023224, "reward_std": 0.16101687587797642, "rewards/accuracy_reward/mean": 0.1316964253783226, "rewards/accuracy_reward/std": 0.32954465970396996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 2091 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 855.6071929931641, "completions/mean_terminated_length": 737.650634765625, "completions/min_length": 361.25, "completions/min_terminated_length": 361.25, "epoch": 0.6248973190949145, "grad_norm": 0.26321545243263245, "kl": 1.9697265625, "learning_rate": 6.467292468600281e-06, "loss": 0.126, "num_tokens": 1018871141.0, "reward": 0.7276786118745804, "reward_std": 0.2136153243482113, "rewards/accuracy_reward/mean": 0.2366071455180645, "rewards/accuracy_reward/std": 0.40879786014556885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04420433798804879, "step": 2092 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 825.8750305175781, "completions/mean_terminated_length": 714.0395812988281, "completions/min_length": 328.25, "completions/min_terminated_length": 328.25, "epoch": 0.6251960271824359, "grad_norm": 0.4420674443244934, "kl": 2.28515625, "learning_rate": 6.4570896200228415e-06, "loss": 0.1347, "num_tokens": 1019315565.0, "reward": 0.7148437649011612, "reward_std": 0.19553978368639946, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.39779242873191833, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 2093 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 867.4308319091797, "completions/mean_terminated_length": 758.8137359619141, "completions/min_length": 222.75, "completions/min_terminated_length": 222.75, "epoch": 0.6254947352699575, "grad_norm": 0.2376125156879425, "kl": 2.392578125, "learning_rate": 6.446890987195842e-06, "loss": 0.1071, "num_tokens": 1019783294.0, "reward": 0.7494420111179352, "reward_std": 0.2263256497681141, "rewards/accuracy_reward/mean": 0.2611607126891613, "rewards/accuracy_reward/std": 0.3902205005288124, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05277834925800562, "step": 2094 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 833.7076263427734, "completions/mean_terminated_length": 729.9937286376953, "completions/min_length": 381.5, "completions/min_terminated_length": 381.5, "epoch": 0.6257934433574789, "grad_norm": 0.26166844367980957, "kl": 2.328125, "learning_rate": 6.436696582254742e-06, "loss": 0.1342, "num_tokens": 1020226731.0, "reward": 0.592075914144516, "reward_std": 0.1268890118226409, "rewards/accuracy_reward/mean": 0.10044642677530646, "rewards/accuracy_reward/std": 0.2219049595296383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 2095 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 875.5580902099609, "completions/mean_terminated_length": 782.9719085693359, "completions/min_length": 402.75, "completions/min_terminated_length": 402.75, "epoch": 0.6260921514450004, "grad_norm": 0.28588834404945374, "kl": 2.494140625, "learning_rate": 6.42650641733e-06, "loss": 0.1249, "num_tokens": 1020694117.0, "reward": 0.6116071715950966, "reward_std": 0.16424890607595444, "rewards/accuracy_reward/mean": 0.12276785634458065, "rewards/accuracy_reward/std": 0.27046966552734375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2096 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 838.5826263427734, "completions/mean_terminated_length": 738.7404479980469, "completions/min_length": 374.75, "completions/min_terminated_length": 374.75, "epoch": 0.6263908595325218, "grad_norm": 0.43516987562179565, "kl": 2.75, "learning_rate": 6.4163205045469975e-06, "loss": 0.1385, "num_tokens": 1021143050.0, "reward": 0.6573660969734192, "reward_std": 0.15901121497154236, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.3695570006966591, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05796953849494457, "step": 2097 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 909.4754791259766, "completions/mean_terminated_length": 805.9677124023438, "completions/min_length": 432.0, "completions/min_terminated_length": 432.0, "epoch": 0.6266895676200434, "grad_norm": 0.3410685062408447, "kl": 2.802734375, "learning_rate": 6.406138856026081e-06, "loss": 0.1334, "num_tokens": 1021634335.0, "reward": 0.5597098469734192, "reward_std": 0.10360706830397248, "rewards/accuracy_reward/mean": 0.07142856949940324, "rewards/accuracy_reward/std": 0.2022436521947384, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918293237686, "step": 2098 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27901785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 804.3661041259766, "completions/mean_terminated_length": 721.0179901123047, "completions/min_length": 373.75, "completions/min_terminated_length": 373.75, "epoch": 0.6269882757075648, "grad_norm": 0.27603405714035034, "kl": 2.59765625, "learning_rate": 6.3959614838825045e-06, "loss": 0.1394, "num_tokens": 1022061523.0, "reward": 0.6199776977300644, "reward_std": 0.11914913356304169, "rewards/accuracy_reward/mean": 0.13169643143191934, "rewards/accuracy_reward/std": 0.3153270222246647, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05243533570319414, "step": 2099 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 858.4799499511719, "completions/mean_terminated_length": 748.2243957519531, "completions/min_length": 307.75, "completions/min_terminated_length": 307.75, "epoch": 0.6272869837950863, "grad_norm": 0.41674932837486267, "kl": 2.984375, "learning_rate": 6.385788400226451e-06, "loss": 0.1576, "num_tokens": 1022510186.0, "reward": 0.6294643133878708, "reward_std": 0.1654108725488186, "rewards/accuracy_reward/mean": 0.1428571380674839, "rewards/accuracy_reward/std": 0.2945270761847496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148982971907, "step": 2100 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 884.8661193847656, "completions/mean_terminated_length": 781.4076080322266, "completions/min_length": 444.5, "completions/min_terminated_length": 444.5, "epoch": 0.6275856918826077, "grad_norm": 0.26620763540267944, "kl": 2.90234375, "learning_rate": 6.375619617162985e-06, "loss": 0.1459, "num_tokens": 1022981678.0, "reward": 0.599888414144516, "reward_std": 0.13778947107493877, "rewards/accuracy_reward/mean": 0.1138392873108387, "rewards/accuracy_reward/std": 0.31727103143930435, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.055549753829836845, "step": 2101 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 835.0134429931641, "completions/mean_terminated_length": 741.5200958251953, "completions/min_length": 274.75, "completions/min_terminated_length": 274.75, "epoch": 0.6278843999701292, "grad_norm": 0.23226092755794525, "kl": 1.7080078125, "learning_rate": 6.365455146792062e-06, "loss": 0.0817, "num_tokens": 1023424996.0, "reward": 0.6489955633878708, "reward_std": 0.14705889858305454, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.35141223669052124, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03606722131371498, "step": 2102 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.26785714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 801.4420013427734, "completions/mean_terminated_length": 725.3744201660156, "completions/min_length": 324.5, "completions/min_terminated_length": 324.5, "epoch": 0.6281831080576507, "grad_norm": 0.303072452545166, "kl": 2.46875, "learning_rate": 6.355295001208504e-06, "loss": 0.1284, "num_tokens": 1023853642.0, "reward": 0.6579241305589676, "reward_std": 0.19426347315311432, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.3648209646344185, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04951908625662327, "step": 2103 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2522321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 814.2879638671875, "completions/mean_terminated_length": 746.7418823242188, "completions/min_length": 392.75, "completions/min_terminated_length": 392.75, "epoch": 0.6284818161451722, "grad_norm": 0.2842417359352112, "kl": 2.576171875, "learning_rate": 6.34513919250199e-06, "loss": 0.1331, "num_tokens": 1024293499.0, "reward": 0.7053571790456772, "reward_std": 0.21083847619593143, "rewards/accuracy_reward/mean": 0.21651786006987095, "rewards/accuracy_reward/std": 0.3926125392317772, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 2104 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.24330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 799.0312805175781, "completions/mean_terminated_length": 727.6469573974609, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "epoch": 0.6287805242326936, "grad_norm": 0.24651901423931122, "kl": 2.572265625, "learning_rate": 6.334987732757028e-06, "loss": 0.1558, "num_tokens": 1024725721.0, "reward": 0.6774553954601288, "reward_std": 0.2091411016881466, "rewards/accuracy_reward/mean": 0.18973213993012905, "rewards/accuracy_reward/std": 0.3721420019865036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05253631342202425, "step": 2105 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33258928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 837.8951263427734, "completions/mean_terminated_length": 744.8452453613281, "completions/min_length": 368.25, "completions/min_terminated_length": 368.25, "epoch": 0.6290792323202151, "grad_norm": 0.39906835556030273, "kl": 2.943359375, "learning_rate": 6.3248406340529665e-06, "loss": 0.1625, "num_tokens": 1025163162.0, "reward": 0.6205357313156128, "reward_std": 0.15090986900031567, "rewards/accuracy_reward/mean": 0.1398809552192688, "rewards/accuracy_reward/std": 0.34517961740493774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.056180731393396854, "step": 2106 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.24553571428571427, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 798.1004791259766, "completions/mean_terminated_length": 724.2238311767578, "completions/min_length": 345.25, "completions/min_terminated_length": 345.25, "epoch": 0.6293779404077365, "grad_norm": 0.5329673290252686, "kl": 2.81640625, "learning_rate": 6.314697908463951e-06, "loss": 0.1798, "num_tokens": 1025587095.0, "reward": 0.6612723618745804, "reward_std": 0.1912886742502451, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.3620685338973999, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05972688551992178, "step": 2107 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 806.5268096923828, "completions/mean_terminated_length": 737.3192901611328, "completions/min_length": 359.75, "completions/min_terminated_length": 359.75, "epoch": 0.6296766484952581, "grad_norm": 0.5198416113853455, "kl": 1.705078125, "learning_rate": 6.3045595680589345e-06, "loss": 0.0826, "num_tokens": 1026022227.0, "reward": 0.635044664144516, "reward_std": 0.13658490404486656, "rewards/accuracy_reward/mean": 0.14285714086145163, "rewards/accuracy_reward/std": 0.32014037296175957, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.03758151177316904, "step": 2108 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.24330357142857142, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 765.0714721679688, "completions/mean_terminated_length": 683.7238922119141, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 0.6299753565827795, "grad_norm": 0.42571958899497986, "kl": 2.6796875, "learning_rate": 6.294425624901638e-06, "loss": 0.1699, "num_tokens": 1026434675.0, "reward": 0.7148437947034836, "reward_std": 0.1698553105816245, "rewards/accuracy_reward/mean": 0.2254464286379516, "rewards/accuracy_reward/std": 0.28933028131723404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05040315631777048, "step": 2109 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 855.2701263427734, "completions/mean_terminated_length": 757.011474609375, "completions/min_length": 321.5, "completions/min_terminated_length": 321.5, "epoch": 0.6302740646703009, "grad_norm": 0.3593372106552124, "kl": 2.87890625, "learning_rate": 6.28429609105057e-06, "loss": 0.1517, "num_tokens": 1026892812.0, "reward": 0.654575914144516, "reward_std": 0.15735303796827793, "rewards/accuracy_reward/mean": 0.16964285215362906, "rewards/accuracy_reward/std": 0.34334199130535126, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330261349678, "rewards/tag_count_reward/std": 0.059475820511579514, "step": 2110 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 819.1920013427734, "completions/mean_terminated_length": 758.8549652099609, "completions/min_length": 398.0, "completions/min_terminated_length": 398.0, "epoch": 0.6305727727578224, "grad_norm": 0.3570272922515869, "kl": 2.212890625, "learning_rate": 6.274170978558971e-06, "loss": 0.1222, "num_tokens": 1027328178.0, "reward": 0.679129496216774, "reward_std": 0.16894493997097015, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3924490138888359, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 2111 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 803.1540679931641, "completions/mean_terminated_length": 734.3506622314453, "completions/min_length": 337.5, "completions/min_terminated_length": 337.5, "epoch": 0.6308714808453438, "grad_norm": 0.3578711152076721, "kl": 2.103515625, "learning_rate": 6.2640502994748375e-06, "loss": 0.1247, "num_tokens": 1027756631.0, "reward": 0.6272321790456772, "reward_std": 0.16373389214277267, "rewards/accuracy_reward/mean": 0.1361607122235, "rewards/accuracy_reward/std": 0.32336602360010147, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 2112 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 817.7344207763672, "completions/mean_terminated_length": 722.3255310058594, "completions/min_length": 329.5, "completions/min_terminated_length": 329.5, "epoch": 0.6311701889328654, "grad_norm": 0.271697074174881, "kl": 2.572265625, "learning_rate": 6.25393406584088e-06, "loss": 0.1415, "num_tokens": 1028197632.0, "reward": 0.6635044813156128, "reward_std": 0.12287902575917542, "rewards/accuracy_reward/mean": 0.1741071417927742, "rewards/accuracy_reward/std": 0.3018750473856926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04820432187989354, "step": 2113 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 877.5714721679688, "completions/mean_terminated_length": 774.3293304443359, "completions/min_length": 344.5, "completions/min_terminated_length": 344.5, "epoch": 0.6314688970203868, "grad_norm": 0.309809148311615, "kl": 2.373046875, "learning_rate": 6.243822289694528e-06, "loss": 0.1241, "num_tokens": 1028680976.0, "reward": 0.6462053954601288, "reward_std": 0.17732747085392475, "rewards/accuracy_reward/mean": 0.15699404617771506, "rewards/accuracy_reward/std": 0.33522531017661095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048634594306349754, "step": 2114 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 832.8839569091797, "completions/mean_terminated_length": 725.4390869140625, "completions/min_length": 251.75, "completions/min_terminated_length": 251.75, "epoch": 0.6317676051079083, "grad_norm": 0.22119984030723572, "kl": 2.28515625, "learning_rate": 6.233714983067893e-06, "loss": 0.1231, "num_tokens": 1029130060.0, "reward": 0.7639509290456772, "reward_std": 0.144870450720191, "rewards/accuracy_reward/mean": 0.2745535671710968, "rewards/accuracy_reward/std": 0.42695244401693344, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04940675385296345, "step": 2115 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 835.5111999511719, "completions/mean_terminated_length": 726.8478546142578, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.6320663131954297, "grad_norm": 0.31809332966804504, "kl": 2.19921875, "learning_rate": 6.223612157987786e-06, "loss": 0.1327, "num_tokens": 1029572833.0, "reward": 0.663504496216774, "reward_std": 0.194478090852499, "rewards/accuracy_reward/mean": 0.1741071455180645, "rewards/accuracy_reward/std": 0.3737003207206726, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04346958827227354, "step": 2116 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 879.4375457763672, "completions/mean_terminated_length": 796.389404296875, "completions/min_length": 360.5, "completions/min_terminated_length": 360.5, "epoch": 0.6323650212829512, "grad_norm": 0.27284541726112366, "kl": 2.478515625, "learning_rate": 6.2135138264756715e-06, "loss": 0.1181, "num_tokens": 1030049909.0, "reward": 0.5853794813156128, "reward_std": 0.14872695319354534, "rewards/accuracy_reward/mean": 0.09821428451687098, "rewards/accuracy_reward/std": 0.29447871446609497, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 2117 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5825892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 930.7701263427734, "completions/mean_terminated_length": 804.9916076660156, "completions/min_length": 301.75, "completions/min_terminated_length": 301.75, "epoch": 0.6326637293704727, "grad_norm": 0.21220925450325012, "kl": 2.14453125, "learning_rate": 6.2034200005476766e-06, "loss": 0.1032, "num_tokens": 1030542110.0, "reward": 0.5524553805589676, "reward_std": 0.10159742087125778, "rewards/accuracy_reward/mean": 0.06250000093132257, "rewards/accuracy_reward/std": 0.18354490958154202, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047784130088984966, "step": 2118 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 888.8281860351562, "completions/mean_terminated_length": 769.1722869873047, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "epoch": 0.6329624374579942, "grad_norm": 0.5005872845649719, "kl": 2.509765625, "learning_rate": 6.1933306922145556e-06, "loss": 0.1173, "num_tokens": 1031007969.0, "reward": 0.5703125298023224, "reward_std": 0.14494582824409008, "rewards/accuracy_reward/mean": 0.08258928777649999, "rewards/accuracy_reward/std": 0.2580745816230774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05127083417028189, "step": 2119 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 879.4353179931641, "completions/mean_terminated_length": 755.8959045410156, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "epoch": 0.6332611455455156, "grad_norm": 0.4162600040435791, "kl": 1.8916015625, "learning_rate": 6.183245913481701e-06, "loss": 0.1122, "num_tokens": 1031475876.0, "reward": 0.6473214477300644, "reward_std": 0.15606204979121685, "rewards/accuracy_reward/mean": 0.1562499962747097, "rewards/accuracy_reward/std": 0.3529594615101814, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03946960438042879, "step": 2120 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 848.0625305175781, "completions/mean_terminated_length": 749.5009002685547, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 0.6335598536330371, "grad_norm": 0.2235799878835678, "kl": 1.82421875, "learning_rate": 6.173165676349103e-06, "loss": 0.0957, "num_tokens": 1031924736.0, "reward": 0.7075893133878708, "reward_std": 0.18131044879555702, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.4094808027148247, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04620361328125, "step": 2121 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.25, "completions/mean_length": 843.7790679931641, "completions/mean_terminated_length": 687.6212005615234, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.6338585617205585, "grad_norm": 0.2724086046218872, "kl": 2.060546875, "learning_rate": 6.163089992811357e-06, "loss": 0.1118, "num_tokens": 1032381261.0, "reward": 0.6729911118745804, "reward_std": 0.18383721634745598, "rewards/accuracy_reward/mean": 0.1830357164144516, "rewards/accuracy_reward/std": 0.38502271473407745, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 2122 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 851.1808471679688, "completions/mean_terminated_length": 735.5711517333984, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.6341572698080801, "grad_norm": 0.478642076253891, "kl": 2.197265625, "learning_rate": 6.153018874857639e-06, "loss": 0.1294, "num_tokens": 1032833646.0, "reward": 0.7042411118745804, "reward_std": 0.19026913680136204, "rewards/accuracy_reward/mean": 0.2165178544819355, "rewards/accuracy_reward/std": 0.41180963814258575, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 2123 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 851.7835388183594, "completions/mean_terminated_length": 732.2019805908203, "completions/min_length": 358.5, "completions/min_terminated_length": 358.5, "epoch": 0.6344559778956015, "grad_norm": 0.21480640769004822, "kl": 1.6640625, "learning_rate": 6.142952334471686e-06, "loss": 0.0756, "num_tokens": 1033286685.0, "reward": 0.578683078289032, "reward_std": 0.11457190848886967, "rewards/accuracy_reward/mean": 0.08705357043072581, "rewards/accuracy_reward/std": 0.2604444921016693, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 2124 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 886.2433471679688, "completions/mean_terminated_length": 769.4469909667969, "completions/min_length": 406.75, "completions/min_terminated_length": 406.75, "epoch": 0.634754685983123, "grad_norm": 0.24882663786411285, "kl": 2.3017578125, "learning_rate": 6.132890383631796e-06, "loss": 0.1088, "num_tokens": 1033756106.0, "reward": 0.6428571790456772, "reward_std": 0.15763731114566326, "rewards/accuracy_reward/mean": 0.15625000139698386, "rewards/accuracy_reward/std": 0.3243621662259102, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05471411347389221, "step": 2125 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39062500000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 858.4330902099609, "completions/mean_terminated_length": 750.4111785888672, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 0.6350533940706444, "grad_norm": 0.2252226024866104, "kl": 2.611328125, "learning_rate": 6.122833034310794e-06, "loss": 0.1505, "num_tokens": 1034217324.0, "reward": 0.6506696790456772, "reward_std": 0.19777612388134003, "rewards/accuracy_reward/mean": 0.16294643096625805, "rewards/accuracy_reward/std": 0.28083329647779465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 2126 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 913.8906707763672, "completions/mean_terminated_length": 802.9526977539062, "completions/min_length": 448.75, "completions/min_terminated_length": 448.75, "epoch": 0.635352102158166, "grad_norm": 0.29493260383605957, "kl": 2.1826171875, "learning_rate": 6.112780298476044e-06, "loss": 0.1184, "num_tokens": 1034707515.0, "reward": 0.6093750298023224, "reward_std": 0.13002244010567665, "rewards/accuracy_reward/mean": 0.12053571618162096, "rewards/accuracy_reward/std": 0.2635918129235506, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050612835213541985, "step": 2127 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 895.0803985595703, "completions/mean_terminated_length": 770.0090026855469, "completions/min_length": 274.0, "completions/min_terminated_length": 274.0, "epoch": 0.6356508102456874, "grad_norm": 0.22415807843208313, "kl": 1.82421875, "learning_rate": 6.102732188089412e-06, "loss": 0.0802, "num_tokens": 1035179743.0, "reward": 0.6311384290456772, "reward_std": 0.1376386433839798, "rewards/accuracy_reward/mean": 0.14062499580904841, "rewards/accuracy_reward/std": 0.31876140832901, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04565368499606848, "step": 2128 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 830.0178833007812, "completions/mean_terminated_length": 715.1603240966797, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "epoch": 0.6359495183332089, "grad_norm": 0.37538769841194153, "kl": 2.162109375, "learning_rate": 6.092688715107265e-06, "loss": 0.1216, "num_tokens": 1035625255.0, "reward": 0.693638414144516, "reward_std": 0.1763404682278633, "rewards/accuracy_reward/mean": 0.2031250037252903, "rewards/accuracy_reward/std": 0.38163869455456734, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2129 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 872.4129791259766, "completions/mean_terminated_length": 759.9634246826172, "completions/min_length": 347.75, "completions/min_terminated_length": 347.75, "epoch": 0.6362482264207303, "grad_norm": 0.3205130696296692, "kl": 2.5703125, "learning_rate": 6.082649891480441e-06, "loss": 0.1457, "num_tokens": 1036086864.0, "reward": 0.6718750298023224, "reward_std": 0.21329206600785255, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3866987004876137, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 2130 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35491071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 820.4687652587891, "completions/mean_terminated_length": 709.1044006347656, "completions/min_length": 315.5, "completions/min_terminated_length": 315.5, "epoch": 0.6365469345082518, "grad_norm": 0.267387717962265, "kl": 2.2734375, "learning_rate": 6.072615729154261e-06, "loss": 0.115, "num_tokens": 1036534546.0, "reward": 0.6183035969734192, "reward_std": 0.1746545135974884, "rewards/accuracy_reward/mean": 0.1294642873108387, "rewards/accuracy_reward/std": 0.32340628653764725, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 2131 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 849.388427734375, "completions/mean_terminated_length": 756.2254486083984, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "epoch": 0.6368456425957733, "grad_norm": 0.19348718225955963, "kl": 1.55078125, "learning_rate": 6.062586240068486e-06, "loss": 0.0852, "num_tokens": 1036988720.0, "reward": 0.6707589626312256, "reward_std": 0.12989302352070808, "rewards/accuracy_reward/mean": 0.17857143143191934, "rewards/accuracy_reward/std": 0.3479445092380047, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04357414972037077, "step": 2132 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2522321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 773.2745819091797, "completions/mean_terminated_length": 692.2015991210938, "completions/min_length": 273.75, "completions/min_terminated_length": 273.75, "epoch": 0.6371443506832948, "grad_norm": 0.27960672974586487, "kl": 2.08203125, "learning_rate": 6.052561436157329e-06, "loss": 0.1221, "num_tokens": 1037405883.0, "reward": 0.6863839626312256, "reward_std": 0.18136587738990784, "rewards/accuracy_reward/mean": 0.1941964253783226, "rewards/accuracy_reward/std": 0.38773029297590256, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 2133 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 805.6205749511719, "completions/mean_terminated_length": 705.02978515625, "completions/min_length": 297.5, "completions/min_terminated_length": 297.5, "epoch": 0.6374430587708162, "grad_norm": 0.20927532017230988, "kl": 2.232421875, "learning_rate": 6.042541329349414e-06, "loss": 0.1261, "num_tokens": 1037836337.0, "reward": 0.607700914144516, "reward_std": 0.10357784852385521, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.319895975291729, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.04348720656707883, "step": 2134 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 836.1339721679688, "completions/mean_terminated_length": 756.9346313476562, "completions/min_length": 405.75, "completions/min_terminated_length": 405.75, "epoch": 0.6377417668583377, "grad_norm": 0.2810899615287781, "kl": 1.95703125, "learning_rate": 6.0325259315677895e-06, "loss": 0.1021, "num_tokens": 1038280701.0, "reward": 0.7723214477300644, "reward_std": 0.16534757055342197, "rewards/accuracy_reward/mean": 0.2790178535506129, "rewards/accuracy_reward/std": 0.4083055779337883, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 2135 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2633928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 814.8973541259766, "completions/mean_terminated_length": 739.5978851318359, "completions/min_length": 390.5, "completions/min_terminated_length": 390.5, "epoch": 0.6380404749458591, "grad_norm": 0.22947020828723907, "kl": 1.837890625, "learning_rate": 6.02251525472989e-06, "loss": 0.1019, "num_tokens": 1038720671.0, "reward": 0.718191996216774, "reward_std": 0.17170702666044235, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.3933755122125149, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.0412445142865181, "step": 2136 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 834.185302734375, "completions/mean_terminated_length": 748.9927520751953, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.6383391830333807, "grad_norm": 0.23079682886600494, "kl": 1.72265625, "learning_rate": 6.0125093107475385e-06, "loss": 0.1036, "num_tokens": 1039178434.0, "reward": 0.7254464477300644, "reward_std": 0.21665141358971596, "rewards/accuracy_reward/mean": 0.23214285727590322, "rewards/accuracy_reward/std": 0.396642342209816, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2137 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2790178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.5, "completions/mean_length": 761.4330596923828, "completions/mean_terminated_length": 666.1892242431641, "completions/min_length": 223.75, "completions/min_terminated_length": 223.75, "epoch": 0.6386378911209021, "grad_norm": 0.3425352871417999, "kl": 2.55859375, "learning_rate": 6.002508111526923e-06, "loss": 0.1475, "num_tokens": 1039599924.0, "reward": 0.8164062947034836, "reward_std": 0.25548355653882027, "rewards/accuracy_reward/mean": 0.32589285634458065, "rewards/accuracy_reward/std": 0.43737658113241196, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047917463816702366, "step": 2138 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 847.8906555175781, "completions/mean_terminated_length": 746.7841491699219, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "epoch": 0.6389365992084236, "grad_norm": 0.2591945230960846, "kl": 1.5927734375, "learning_rate": 5.9925116689685925e-06, "loss": 0.0871, "num_tokens": 1040047107.0, "reward": 0.6579241305589676, "reward_std": 0.13597978465259075, "rewards/accuracy_reward/mean": 0.1651785746216774, "rewards/accuracy_reward/std": 0.36068839579820633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.038913180120289326, "step": 2139 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 808.3683471679688, "completions/mean_terminated_length": 698.3974304199219, "completions/min_length": 328.25, "completions/min_terminated_length": 328.25, "epoch": 0.639235307295945, "grad_norm": 0.28322702646255493, "kl": 2.556640625, "learning_rate": 5.982519994967423e-06, "loss": 0.1464, "num_tokens": 1040478536.0, "reward": 0.732700914144516, "reward_std": 0.21105441451072693, "rewards/accuracy_reward/mean": 0.2433035783469677, "rewards/accuracy_reward/std": 0.40895265340805054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.049606312066316605, "step": 2140 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 805.2745971679688, "completions/mean_terminated_length": 705.0582580566406, "completions/min_length": 332.75, "completions/min_terminated_length": 332.75, "epoch": 0.6395340153834665, "grad_norm": 0.2533819377422333, "kl": 1.7783203125, "learning_rate": 5.97253310141263e-06, "loss": 0.1042, "num_tokens": 1040910947.0, "reward": 0.6635044813156128, "reward_std": 0.11389441159553826, "rewards/accuracy_reward/mean": 0.1796875037252903, "rewards/accuracy_reward/std": 0.3153500333428383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616156578064, "rewards/tag_count_reward/std": 0.036427486687898636, "step": 2141 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 823.1183471679688, "completions/mean_terminated_length": 711.0507659912109, "completions/min_length": 302.25, "completions/min_terminated_length": 302.25, "epoch": 0.639832723470988, "grad_norm": 0.4681197702884674, "kl": 2.44921875, "learning_rate": 5.962551000187728e-06, "loss": 0.1244, "num_tokens": 1041356376.0, "reward": 0.6456473618745804, "reward_std": 0.17876120028086007, "rewards/accuracy_reward/mean": 0.1540178544819355, "rewards/accuracy_reward/std": 0.3000005632638931, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.04348720656707883, "step": 2142 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.5, "completions/mean_length": 856.0826263427734, "completions/mean_terminated_length": 737.2169189453125, "completions/min_length": 409.75, "completions/min_terminated_length": 409.75, "epoch": 0.6401314315585095, "grad_norm": 0.28819650411605835, "kl": 3.31640625, "learning_rate": 5.952573703170548e-06, "loss": 0.1788, "num_tokens": 1041817965.0, "reward": 0.668526828289032, "reward_std": 0.18804392404854298, "rewards/accuracy_reward/mean": 0.1830357159487903, "rewards/accuracy_reward/std": 0.31379294767975807, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05796953663229942, "step": 2143 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 820.9085235595703, "completions/mean_terminated_length": 708.7462921142578, "completions/min_length": 294.25, "completions/min_terminated_length": 294.25, "epoch": 0.6404301396460309, "grad_norm": 0.3597376048564911, "kl": 2.87890625, "learning_rate": 5.942601222233179e-06, "loss": 0.1558, "num_tokens": 1042250756.0, "reward": 0.7025669813156128, "reward_std": 0.16656335070729256, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.40708979219198227, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.057296221144497395, "step": 2144 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 855.5379791259766, "completions/mean_terminated_length": 728.5850677490234, "completions/min_length": 352.5, "completions/min_terminated_length": 352.5, "epoch": 0.6407288477335524, "grad_norm": 0.19931112229824066, "kl": 1.298828125, "learning_rate": 5.932633569242e-06, "loss": 0.0741, "num_tokens": 1042708101.0, "reward": 0.6690848469734192, "reward_std": 0.17795102670788765, "rewards/accuracy_reward/mean": 0.1741071450524032, "rewards/accuracy_reward/std": 0.3507891893386841, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.029593830928206444, "step": 2145 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 834.4018402099609, "completions/mean_terminated_length": 733.5020141601562, "completions/min_length": 384.0, "completions/min_terminated_length": 384.0, "epoch": 0.6410275558210738, "grad_norm": 0.4323252737522125, "kl": 1.724609375, "learning_rate": 5.922670756057633e-06, "loss": 0.1005, "num_tokens": 1043151705.0, "reward": 0.5943080633878708, "reward_std": 0.1470999028533697, "rewards/accuracy_reward/mean": 0.10267857206054032, "rewards/accuracy_reward/std": 0.2728181853890419, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 2146 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 822.9821624755859, "completions/mean_terminated_length": 712.2996978759766, "completions/min_length": 293.25, "completions/min_terminated_length": 293.25, "epoch": 0.6413262639085954, "grad_norm": 0.3265610933303833, "kl": 1.6591796875, "learning_rate": 5.912712794534954e-06, "loss": 0.0899, "num_tokens": 1043589521.0, "reward": 0.6177455633878708, "reward_std": 0.14281835034489632, "rewards/accuracy_reward/mean": 0.1283482122235, "rewards/accuracy_reward/std": 0.3093285635113716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03606722131371498, "step": 2147 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 864.0558319091797, "completions/mean_terminated_length": 767.1316223144531, "completions/min_length": 277.5, "completions/min_terminated_length": 277.5, "epoch": 0.6416249719961168, "grad_norm": 0.28051164746284485, "kl": 1.599609375, "learning_rate": 5.902759696523046e-06, "loss": 0.0771, "num_tokens": 1044042890.0, "reward": 0.5279017984867096, "reward_std": 0.10573727078735828, "rewards/accuracy_reward/mean": 0.035714287078008056, "rewards/accuracy_reward/std": 0.17544529400765896, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 2148 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 844.7299499511719, "completions/mean_terminated_length": 718.5212554931641, "completions/min_length": 331.0, "completions/min_terminated_length": 331.0, "epoch": 0.6419236800836383, "grad_norm": 0.3870527744293213, "kl": 1.5380859375, "learning_rate": 5.89281147386523e-06, "loss": 0.0961, "num_tokens": 1044504305.0, "reward": 0.7539062798023224, "reward_std": 0.17894934676587582, "rewards/accuracy_reward/mean": 0.2611607164144516, "rewards/accuracy_reward/std": 0.43006056547164917, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 2149 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 870.4486846923828, "completions/mean_terminated_length": 755.8349761962891, "completions/min_length": 276.0, "completions/min_terminated_length": 276.0, "epoch": 0.6422223881711597, "grad_norm": 0.2805444002151489, "kl": 2.484375, "learning_rate": 5.882868138399e-06, "loss": 0.1261, "num_tokens": 1044968554.0, "reward": 0.5937500298023224, "reward_std": 0.133919857442379, "rewards/accuracy_reward/mean": 0.10714285867288709, "rewards/accuracy_reward/std": 0.2901579774916172, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 2150 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 871.8995971679688, "completions/mean_terminated_length": 763.6004791259766, "completions/min_length": 325.75, "completions/min_terminated_length": 325.75, "epoch": 0.6425210962586813, "grad_norm": 0.29495373368263245, "kl": 2.01953125, "learning_rate": 5.872929701956054e-06, "loss": 0.1053, "num_tokens": 1045431069.0, "reward": 0.662388414144516, "reward_std": 0.1782055888324976, "rewards/accuracy_reward/mean": 0.17187499743886292, "rewards/accuracy_reward/std": 0.3339142557233572, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047120303846895695, "step": 2151 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 840.7388916015625, "completions/mean_terminated_length": 723.2775573730469, "completions/min_length": 268.25, "completions/min_terminated_length": 268.25, "epoch": 0.6428198043462027, "grad_norm": 0.27802178263664246, "kl": 1.8271484375, "learning_rate": 5.86299617636225e-06, "loss": 0.1185, "num_tokens": 1045884456.0, "reward": 0.6869419813156128, "reward_std": 0.17176639661192894, "rewards/accuracy_reward/mean": 0.19419642724096775, "rewards/accuracy_reward/std": 0.3861265629529953, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 2152 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 838.4286041259766, "completions/mean_terminated_length": 706.1917877197266, "completions/min_length": 293.25, "completions/min_terminated_length": 293.25, "epoch": 0.6431185124337241, "grad_norm": 0.3238450884819031, "kl": 1.45703125, "learning_rate": 5.853067573437612e-06, "loss": 0.0989, "num_tokens": 1046330120.0, "reward": 0.6657366305589676, "reward_std": 0.15401527658104897, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.35578572005033493, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521269638091326, "step": 2153 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 901.4911041259766, "completions/mean_terminated_length": 795.1269378662109, "completions/min_length": 462.75, "completions/min_terminated_length": 462.75, "epoch": 0.6434172205212456, "grad_norm": 0.4040040671825409, "kl": 1.99609375, "learning_rate": 5.843143904996294e-06, "loss": 0.1044, "num_tokens": 1046814052.0, "reward": 0.5468750223517418, "reward_std": 0.09435372706502676, "rewards/accuracy_reward/mean": 0.05803571385331452, "rewards/accuracy_reward/std": 0.1807586755603552, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2154 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 865.2143249511719, "completions/mean_terminated_length": 733.1423034667969, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "epoch": 0.643715928608767, "grad_norm": 0.38887616991996765, "kl": 2.24609375, "learning_rate": 5.833225182846587e-06, "loss": 0.1046, "num_tokens": 1047272916.0, "reward": 0.6830357611179352, "reward_std": 0.20301125943660736, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.394795224070549, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 2155 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 897.0580902099609, "completions/mean_terminated_length": 790.3869171142578, "completions/min_length": 300.75, "completions/min_terminated_length": 300.75, "epoch": 0.6440146366962886, "grad_norm": 0.6775860786437988, "kl": 0.99267578125, "learning_rate": 5.823311418790894e-06, "loss": 0.051, "num_tokens": 1047749422.0, "reward": 0.7047991305589676, "reward_std": 0.1935398243367672, "rewards/accuracy_reward/mean": 0.20982142724096775, "rewards/accuracy_reward/std": 0.37764905393123627, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 2156 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 893.7612152099609, "completions/mean_terminated_length": 745.8351745605469, "completions/min_length": 332.25, "completions/min_terminated_length": 332.25, "epoch": 0.64431334478381, "grad_norm": 0.4289032220840454, "kl": 2.267578125, "learning_rate": 5.813402624625722e-06, "loss": 0.1159, "num_tokens": 1048225987.0, "reward": 0.5496652200818062, "reward_std": 0.1176172187551856, "rewards/accuracy_reward/mean": 0.06249999860301614, "rewards/accuracy_reward/std": 0.18829556554555893, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 2157 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 873.8817291259766, "completions/mean_terminated_length": 761.620361328125, "completions/min_length": 326.75, "completions/min_terminated_length": 326.75, "epoch": 0.6446120528713315, "grad_norm": 0.35191917419433594, "kl": 2.244140625, "learning_rate": 5.803498812141656e-06, "loss": 0.1334, "num_tokens": 1048686638.0, "reward": 0.6183036118745804, "reward_std": 0.1385557409375906, "rewards/accuracy_reward/mean": 0.1294642835855484, "rewards/accuracy_reward/std": 0.32710450142621994, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050612835213541985, "step": 2158 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 861.1451263427734, "completions/mean_terminated_length": 720.0050354003906, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "epoch": 0.6449107609588529, "grad_norm": 0.31804847717285156, "kl": 2.470703125, "learning_rate": 5.793599993123363e-06, "loss": 0.1257, "num_tokens": 1049147167.0, "reward": 0.5887276977300644, "reward_std": 0.10584144294261932, "rewards/accuracy_reward/mean": 0.10044642980210483, "rewards/accuracy_reward/std": 0.24042225815355778, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 2159 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 886.5803985595703, "completions/mean_terminated_length": 753.1303863525391, "completions/min_length": 367.75, "completions/min_terminated_length": 367.75, "epoch": 0.6452094690463744, "grad_norm": 0.2953731417655945, "kl": 2.1484375, "learning_rate": 5.783706179349553e-06, "loss": 0.116, "num_tokens": 1049610035.0, "reward": 0.7087053805589676, "reward_std": 0.2286541908979416, "rewards/accuracy_reward/mean": 0.21875, "rewards/accuracy_reward/std": 0.4062065854668617, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048092021606862545, "step": 2160 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.484375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 866.0982513427734, "completions/mean_terminated_length": 723.0228881835938, "completions/min_length": 382.25, "completions/min_terminated_length": 382.25, "epoch": 0.6455081771338959, "grad_norm": 0.3098628520965576, "kl": 2.453125, "learning_rate": 5.773817382593008e-06, "loss": 0.1378, "num_tokens": 1050071887.0, "reward": 0.7460937947034836, "reward_std": 0.2185831293463707, "rewards/accuracy_reward/mean": 0.2566964253783226, "rewards/accuracy_reward/std": 0.4161071181297302, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2161 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 865.1004791259766, "completions/mean_terminated_length": 742.6793060302734, "completions/min_length": 274.5, "completions/min_terminated_length": 274.5, "epoch": 0.6458068852214174, "grad_norm": 0.4238694906234741, "kl": 3.138671875, "learning_rate": 5.763933614620507e-06, "loss": 0.1641, "num_tokens": 1050528124.0, "reward": 0.6037946715950966, "reward_std": 0.15317885018885136, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.26828905940055847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05698250140994787, "step": 2162 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 878.2344055175781, "completions/mean_terminated_length": 746.0082855224609, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 0.6461055933089388, "grad_norm": 0.339338481426239, "kl": 2.5498046875, "learning_rate": 5.754054887192871e-06, "loss": 0.1215, "num_tokens": 1051004373.0, "reward": 0.7081473469734192, "reward_std": 0.19052119553089142, "rewards/accuracy_reward/mean": 0.21875000605359674, "rewards/accuracy_reward/std": 0.34873754158616066, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04740747855976224, "step": 2163 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 855.8594207763672, "completions/mean_terminated_length": 715.9223175048828, "completions/min_length": 344.5, "completions/min_terminated_length": 344.5, "epoch": 0.6464043013964603, "grad_norm": 0.35764917731285095, "kl": 3.072265625, "learning_rate": 5.7441812120649065e-06, "loss": 0.1804, "num_tokens": 1051457814.0, "reward": 0.6082589477300644, "reward_std": 0.14571385458111763, "rewards/accuracy_reward/mean": 0.12276785774156451, "rewards/accuracy_reward/std": 0.3123571425676346, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05652147252112627, "step": 2164 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37946428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 851.6428985595703, "completions/mean_terminated_length": 749.8881683349609, "completions/min_length": 267.25, "completions/min_terminated_length": 267.25, "epoch": 0.6467030094839817, "grad_norm": 0.3786013722419739, "kl": 2.673828125, "learning_rate": 5.7343126009854215e-06, "loss": 0.1244, "num_tokens": 1051908854.0, "reward": 0.6768973469734192, "reward_std": 0.14953167364001274, "rewards/accuracy_reward/mean": 0.1874999962747097, "rewards/accuracy_reward/std": 0.38628973811864853, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.049606312066316605, "step": 2165 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 879.810302734375, "completions/mean_terminated_length": 784.1395263671875, "completions/min_length": 348.75, "completions/min_terminated_length": 348.75, "epoch": 0.6470017175715033, "grad_norm": 0.3410530090332031, "kl": 3.22265625, "learning_rate": 5.724449065697182e-06, "loss": 0.1643, "num_tokens": 1052383793.0, "reward": 0.7243303805589676, "reward_std": 0.2011539675295353, "rewards/accuracy_reward/mean": 0.23883928544819355, "rewards/accuracy_reward/std": 0.40801382064819336, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058577682822942734, "step": 2166 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 832.7879791259766, "completions/mean_terminated_length": 739.6582489013672, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.6473004256590247, "grad_norm": 0.2990397810935974, "kl": 3.212890625, "learning_rate": 5.714590617936931e-06, "loss": 0.1801, "num_tokens": 1052833954.0, "reward": 0.6579241305589676, "reward_std": 0.17211581021547318, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.3563019521534443, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.056990127079188824, "step": 2167 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 815.6652221679688, "completions/mean_terminated_length": 743.0086822509766, "completions/min_length": 443.75, "completions/min_terminated_length": 443.75, "epoch": 0.6475991337465462, "grad_norm": 0.5259183645248413, "kl": 1.7265625, "learning_rate": 5.704737269435346e-06, "loss": 0.0832, "num_tokens": 1053272844.0, "reward": 0.6802455633878708, "reward_std": 0.19363340316340327, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3213172033429146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 2168 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2901785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 801.0089569091797, "completions/mean_terminated_length": 709.2802429199219, "completions/min_length": 294.75, "completions/min_terminated_length": 294.75, "epoch": 0.6478978418340676, "grad_norm": 0.29253026843070984, "kl": 3.01953125, "learning_rate": 5.694889031917047e-06, "loss": 0.1707, "num_tokens": 1053703712.0, "reward": 0.6099330633878708, "reward_std": 0.10626536421477795, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.28014299273490906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.059022306464612484, "step": 2169 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 851.0893249511719, "completions/mean_terminated_length": 770.1725463867188, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "epoch": 0.6481965499215891, "grad_norm": 0.34402158856391907, "kl": 2.2890625, "learning_rate": 5.685045917100568e-06, "loss": 0.1157, "num_tokens": 1054155288.0, "reward": 0.640066996216774, "reward_std": 0.10943141020834446, "rewards/accuracy_reward/mean": 0.1495535671710968, "rewards/accuracy_reward/std": 0.3575339615345001, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2170 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 858.1339569091797, "completions/mean_terminated_length": 756.2347717285156, "completions/min_length": 425.5, "completions/min_terminated_length": 425.5, "epoch": 0.6484952580091106, "grad_norm": 0.4142211675643921, "kl": 3.078125, "learning_rate": 5.675207936698337e-06, "loss": 0.1764, "num_tokens": 1054614516.0, "reward": 0.6534598618745804, "reward_std": 0.186203982681036, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.372247651219368, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.060302252881228924, "step": 2171 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 826.3170166015625, "completions/mean_terminated_length": 746.6930694580078, "completions/min_length": 264.5, "completions/min_terminated_length": 264.5, "epoch": 0.6487939660966321, "grad_norm": 0.2330906093120575, "kl": 2.3203125, "learning_rate": 5.6653751024166925e-06, "loss": 0.1175, "num_tokens": 1055057586.0, "reward": 0.7500000149011612, "reward_std": 0.203892033547163, "rewards/accuracy_reward/mean": 0.2611607201397419, "rewards/accuracy_reward/std": 0.42197275906801224, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051717888563871384, "step": 2172 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.5, "completions/mean_length": 833.6183319091797, "completions/mean_terminated_length": 723.0895233154297, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 0.6490926741841535, "grad_norm": 0.27153515815734863, "kl": 2.55078125, "learning_rate": 5.65554742595583e-06, "loss": 0.1499, "num_tokens": 1055501527.0, "reward": 0.6065848469734192, "reward_std": 0.15159940160810947, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.3212187811732292, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 2173 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 826.4219055175781, "completions/mean_terminated_length": 715.3959350585938, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "epoch": 0.649391382271675, "grad_norm": 0.28946489095687866, "kl": 2.501953125, "learning_rate": 5.64572491900982e-06, "loss": 0.1317, "num_tokens": 1055942148.0, "reward": 0.6819196939468384, "reward_std": 0.12689932622015476, "rewards/accuracy_reward/mean": 0.19419642724096775, "rewards/accuracy_reward/std": 0.35920844972133636, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 2174 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 820.654052734375, "completions/mean_terminated_length": 731.7371978759766, "completions/min_length": 378.5, "completions/min_terminated_length": 378.5, "epoch": 0.6496900903591964, "grad_norm": 0.48522132635116577, "kl": 2.451171875, "learning_rate": 5.635907593266578e-06, "loss": 0.1419, "num_tokens": 1056388249.0, "reward": 0.7265625298023224, "reward_std": 0.21202743984758854, "rewards/accuracy_reward/mean": 0.2388392873108387, "rewards/accuracy_reward/std": 0.40539248287677765, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05394930951297283, "step": 2175 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 838.6942443847656, "completions/mean_terminated_length": 744.020263671875, "completions/min_length": 330.75, "completions/min_terminated_length": 330.75, "epoch": 0.649988798446718, "grad_norm": 0.2651795446872711, "kl": 3.158203125, "learning_rate": 5.6260954604078585e-06, "loss": 0.1629, "num_tokens": 1056837200.0, "reward": 0.662946455180645, "reward_std": 0.17830380517989397, "rewards/accuracy_reward/mean": 0.1785714328289032, "rewards/accuracy_reward/std": 0.3119639679789543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.06077473238110542, "step": 2176 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 867.0893249511719, "completions/mean_terminated_length": 767.9218139648438, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "epoch": 0.6502875065342394, "grad_norm": 0.41712963581085205, "kl": 2.583984375, "learning_rate": 5.616288532109225e-06, "loss": 0.1314, "num_tokens": 1057301064.0, "reward": 0.5652901977300644, "reward_std": 0.15388713218271732, "rewards/accuracy_reward/mean": 0.07589285634458065, "rewards/accuracy_reward/std": 0.25832121074199677, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04868226684629917, "step": 2177 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 869.763427734375, "completions/mean_terminated_length": 754.2233428955078, "completions/min_length": 403.0, "completions/min_terminated_length": 403.0, "epoch": 0.6505862146217609, "grad_norm": 0.3896203339099884, "kl": 2.94921875, "learning_rate": 5.606486820040064e-06, "loss": 0.1428, "num_tokens": 1057756398.0, "reward": 0.5809152126312256, "reward_std": 0.13498163037002087, "rewards/accuracy_reward/mean": 0.09375000023283064, "rewards/accuracy_reward/std": 0.25370556488633156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651902794838, "rewards/tag_count_reward/std": 0.05506479926407337, "step": 2178 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 829.3973693847656, "completions/mean_terminated_length": 720.6643829345703, "completions/min_length": 279.75, "completions/min_terminated_length": 279.75, "epoch": 0.6508849227092823, "grad_norm": 0.31220388412475586, "kl": 2.96484375, "learning_rate": 5.596690335863542e-06, "loss": 0.1645, "num_tokens": 1058206704.0, "reward": 0.7823661118745804, "reward_std": 0.2239377275109291, "rewards/accuracy_reward/mean": 0.3046875, "rewards/accuracy_reward/std": 0.46138273924589157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 2179 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 843.7634124755859, "completions/mean_terminated_length": 742.4970245361328, "completions/min_length": 351.0, "completions/min_terminated_length": 351.0, "epoch": 0.6511836307968039, "grad_norm": 0.4642629027366638, "kl": 2.7109375, "learning_rate": 5.586899091236613e-06, "loss": 0.1434, "num_tokens": 1058654742.0, "reward": 0.6406250298023224, "reward_std": 0.22437116131186485, "rewards/accuracy_reward/mean": 0.15848213899880648, "rewards/accuracy_reward/std": 0.34184209257364273, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05275180237367749, "step": 2180 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 895.904052734375, "completions/mean_terminated_length": 807.3754272460938, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "epoch": 0.6514823388843253, "grad_norm": 0.2558112144470215, "kl": 2.423828125, "learning_rate": 5.5771130978099896e-06, "loss": 0.1261, "num_tokens": 1059136603.0, "reward": 0.6255580633878708, "reward_std": 0.11262869369238615, "rewards/accuracy_reward/mean": 0.13616071734577417, "rewards/accuracy_reward/std": 0.2646168991923332, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.0486822659149766, "step": 2181 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 829.9531555175781, "completions/mean_terminated_length": 752.3571166992188, "completions/min_length": 311.75, "completions/min_terminated_length": 311.75, "epoch": 0.6517810469718468, "grad_norm": 0.26561397314071655, "kl": 2.5771484375, "learning_rate": 5.56733236722814e-06, "loss": 0.1383, "num_tokens": 1059574886.0, "reward": 0.5719866305589676, "reward_std": 0.10139908827841282, "rewards/accuracy_reward/mean": 0.0825892852153629, "rewards/accuracy_reward/std": 0.23432132601737976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04154945630580187, "step": 2182 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 826.2522583007812, "completions/mean_terminated_length": 717.7389373779297, "completions/min_length": 232.5, "completions/min_terminated_length": 232.5, "epoch": 0.6520797550593682, "grad_norm": 0.3435627520084381, "kl": 2.53125, "learning_rate": 5.5575569111292725e-06, "loss": 0.1417, "num_tokens": 1060020567.0, "reward": 0.654575914144516, "reward_std": 0.1751824077218771, "rewards/accuracy_reward/mean": 0.1651785746216774, "rewards/accuracy_reward/std": 0.3707808256149292, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.0494418740272522, "step": 2183 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 853.3861999511719, "completions/mean_terminated_length": 760.0145568847656, "completions/min_length": 376.0, "completions/min_terminated_length": 376.0, "epoch": 0.6523784631468897, "grad_norm": 0.35925158858299255, "kl": 2.49609375, "learning_rate": 5.54778674114532e-06, "loss": 0.1505, "num_tokens": 1060475892.0, "reward": 0.6450893133878708, "reward_std": 0.19400140270590782, "rewards/accuracy_reward/mean": 0.15625000465661287, "rewards/accuracy_reward/std": 0.33470065891742706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05112060345709324, "step": 2184 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 862.0156707763672, "completions/mean_terminated_length": 784.1348266601562, "completions/min_length": 217.5, "completions/min_terminated_length": 217.5, "epoch": 0.6526771712344112, "grad_norm": 0.19930830597877502, "kl": 1.26953125, "learning_rate": 5.5380218689019125e-06, "loss": 0.0622, "num_tokens": 1060931819.0, "reward": 0.6356027126312256, "reward_std": 0.12891317903995514, "rewards/accuracy_reward/mean": 0.14062500093132257, "rewards/accuracy_reward/std": 0.32874395698308945, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03010128252208233, "step": 2185 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39955357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 840.9442291259766, "completions/mean_terminated_length": 723.1606903076172, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "epoch": 0.6529758793219327, "grad_norm": 0.3084237277507782, "kl": 2.8828125, "learning_rate": 5.5282623060183945e-06, "loss": 0.1443, "num_tokens": 1061379602.0, "reward": 0.6875000447034836, "reward_std": 0.1537008099257946, "rewards/accuracy_reward/mean": 0.21391369216144085, "rewards/accuracy_reward/std": 0.3896484896540642, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.0599466897547245, "step": 2186 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 844.6518249511719, "completions/mean_terminated_length": 727.511474609375, "completions/min_length": 346.25, "completions/min_terminated_length": 346.25, "epoch": 0.6532745874094541, "grad_norm": 0.2570800185203552, "kl": 2.44140625, "learning_rate": 5.518508064107776e-06, "loss": 0.1261, "num_tokens": 1061824166.0, "reward": 0.6640625298023224, "reward_std": 0.13930813781917095, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3860680088400841, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.053750067949295044, "step": 2187 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 816.3973541259766, "completions/mean_terminated_length": 703.0414428710938, "completions/min_length": 288.5, "completions/min_terminated_length": 288.5, "epoch": 0.6535732954969756, "grad_norm": 0.5341629981994629, "kl": 2.138671875, "learning_rate": 5.508759154776747e-06, "loss": 0.1394, "num_tokens": 1062266712.0, "reward": 0.6110491305589676, "reward_std": 0.112972775939852, "rewards/accuracy_reward/mean": 0.12053571455180645, "rewards/accuracy_reward/std": 0.269013449549675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 2188 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 804.4174499511719, "completions/mean_terminated_length": 687.5, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.653872003584497, "grad_norm": 0.4245312213897705, "kl": 2.3701171875, "learning_rate": 5.499015589625649e-06, "loss": 0.1485, "num_tokens": 1062699139.0, "reward": 0.6473214626312256, "reward_std": 0.16395106725394726, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.3609217628836632, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04896552674472332, "step": 2189 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 834.7723541259766, "completions/mean_terminated_length": 718.5702209472656, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.6541707116720186, "grad_norm": 0.3912639021873474, "kl": 2.53515625, "learning_rate": 5.489277380248468e-06, "loss": 0.14, "num_tokens": 1063143613.0, "reward": 0.688058078289032, "reward_std": 0.1348101794719696, "rewards/accuracy_reward/mean": 0.1986607126891613, "rewards/accuracy_reward/std": 0.3950062692165375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 2190 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 873.747802734375, "completions/mean_terminated_length": 765.1842803955078, "completions/min_length": 349.0, "completions/min_terminated_length": 349.0, "epoch": 0.65446941975954, "grad_norm": 0.4152168035507202, "kl": 1.7900390625, "learning_rate": 5.479544538232804e-06, "loss": 0.0719, "num_tokens": 1063608012.0, "reward": 0.680245578289032, "reward_std": 0.14357512816786766, "rewards/accuracy_reward/mean": 0.18750000093132257, "rewards/accuracy_reward/std": 0.364654041826725, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03461915533989668, "step": 2191 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43749999999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 888.3638763427734, "completions/mean_terminated_length": 790.0291748046875, "completions/min_length": 468.75, "completions/min_terminated_length": 468.75, "epoch": 0.6547681278470615, "grad_norm": 0.3616848289966583, "kl": 1.580078125, "learning_rate": 5.469817075159887e-06, "loss": 0.0856, "num_tokens": 1064078063.0, "reward": 0.6367187798023224, "reward_std": 0.12800844386219978, "rewards/accuracy_reward/mean": 0.1514136865735054, "rewards/accuracy_reward/std": 0.3530559837818146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 2192 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43303571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 864.1495971679688, "completions/mean_terminated_length": 748.09228515625, "completions/min_length": 319.5, "completions/min_terminated_length": 319.5, "epoch": 0.6550668359345829, "grad_norm": 0.5591179728507996, "kl": 3.2265625, "learning_rate": 5.460095002604533e-06, "loss": 0.1554, "num_tokens": 1064541650.0, "reward": 0.6696428954601288, "reward_std": 0.21328091993927956, "rewards/accuracy_reward/mean": 0.1830357164144516, "rewards/accuracy_reward/std": 0.352908868342638, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05543891713023186, "step": 2193 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 854.1808471679688, "completions/mean_terminated_length": 754.2597351074219, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 0.6553655440221045, "grad_norm": 0.3072448670864105, "kl": 2.015625, "learning_rate": 5.450378332135157e-06, "loss": 0.0917, "num_tokens": 1064995123.0, "reward": 0.6858259290456772, "reward_std": 0.17997172474861145, "rewards/accuracy_reward/mean": 0.206101194024086, "rewards/accuracy_reward/std": 0.37904810160398483, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 2194 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 862.5580749511719, "completions/mean_terminated_length": 769.5966796875, "completions/min_length": 407.0, "completions/min_terminated_length": 407.0, "epoch": 0.6556642521096259, "grad_norm": 0.5854371786117554, "kl": 3.09765625, "learning_rate": 5.44066707531373e-06, "loss": 0.1516, "num_tokens": 1065452429.0, "reward": 0.5881696790456772, "reward_std": 0.15518794022500515, "rewards/accuracy_reward/mean": 0.10044643096625805, "rewards/accuracy_reward/std": 0.29379379749298096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05349547974765301, "step": 2195 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 860.4799346923828, "completions/mean_terminated_length": 771.2843780517578, "completions/min_length": 343.75, "completions/min_terminated_length": 343.75, "epoch": 0.6559629601971473, "grad_norm": 0.251130610704422, "kl": 1.76220703125, "learning_rate": 5.430961243695794e-06, "loss": 0.087, "num_tokens": 1065910340.0, "reward": 0.5781250149011612, "reward_std": 0.12850962206721306, "rewards/accuracy_reward/mean": 0.08482142724096775, "rewards/accuracy_reward/std": 0.2733529955148697, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03475248906761408, "step": 2196 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 855.3036041259766, "completions/mean_terminated_length": 755.3196258544922, "completions/min_length": 325.25, "completions/min_terminated_length": 325.25, "epoch": 0.6562616682846688, "grad_norm": 0.5577090978622437, "kl": 1.7666015625, "learning_rate": 5.421260848830432e-06, "loss": 0.0929, "num_tokens": 1066375100.0, "reward": 0.6568080633878708, "reward_std": 0.12181122042238712, "rewards/accuracy_reward/mean": 0.1651785708963871, "rewards/accuracy_reward/std": 0.334715873003006, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 2197 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 838.7009124755859, "completions/mean_terminated_length": 762.0066986083984, "completions/min_length": 393.25, "completions/min_terminated_length": 393.25, "epoch": 0.6565603763721902, "grad_norm": 0.20650388300418854, "kl": 1.994140625, "learning_rate": 5.41156590226026e-06, "loss": 0.0997, "num_tokens": 1066830038.0, "reward": 0.6116071790456772, "reward_std": 0.1678749080747366, "rewards/accuracy_reward/mean": 0.12834821362048388, "rewards/accuracy_reward/std": 0.32392654567956924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 2198 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35491071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 861.7433319091797, "completions/mean_terminated_length": 772.9850158691406, "completions/min_length": 454.0, "completions/min_terminated_length": 454.0, "epoch": 0.6568590844597118, "grad_norm": 0.33435529470443726, "kl": 2.65234375, "learning_rate": 5.401876415521402e-06, "loss": 0.1483, "num_tokens": 1067285683.0, "reward": 0.6841518133878708, "reward_std": 0.22098411619663239, "rewards/accuracy_reward/mean": 0.19642857275903225, "rewards/accuracy_reward/std": 0.3815421685576439, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.052645014598965645, "step": 2199 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2767857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 811.8683471679688, "completions/mean_terminated_length": 729.6355743408203, "completions/min_length": 372.75, "completions/min_terminated_length": 372.75, "epoch": 0.6571577925472332, "grad_norm": 0.3008614778518677, "kl": 1.828125, "learning_rate": 5.392192400143498e-06, "loss": 0.1131, "num_tokens": 1067718136.0, "reward": 0.710379496216774, "reward_std": 0.13915345002897084, "rewards/accuracy_reward/mean": 0.2187499962747097, "rewards/accuracy_reward/std": 0.3269064351916313, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 2200 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 878.8995971679688, "completions/mean_terminated_length": 770.5674438476562, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "epoch": 0.6574565006347547, "grad_norm": 0.6239707469940186, "kl": 1.716796875, "learning_rate": 5.382513867649663e-06, "loss": 0.0986, "num_tokens": 1068182699.0, "reward": 0.5993303805589676, "reward_std": 0.15192077308893204, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.25922873616218567, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2201 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 800.1942291259766, "completions/mean_terminated_length": 677.9903411865234, "completions/min_length": 310.25, "completions/min_terminated_length": 310.25, "epoch": 0.6577552087222761, "grad_norm": 0.33162975311279297, "kl": 1.396484375, "learning_rate": 5.3728408295565e-06, "loss": 0.0777, "num_tokens": 1068614322.0, "reward": 0.6863839477300644, "reward_std": 0.1376385148614645, "rewards/accuracy_reward/mean": 0.19196428265422583, "rewards/accuracy_reward/std": 0.37011873722076416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 2202 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 854.044677734375, "completions/mean_terminated_length": 773.2757110595703, "completions/min_length": 292.75, "completions/min_terminated_length": 292.75, "epoch": 0.6580539168097976, "grad_norm": 0.5122960209846497, "kl": 1.7138671875, "learning_rate": 5.36317329737407e-06, "loss": 0.0841, "num_tokens": 1069070182.0, "reward": 0.7014509290456772, "reward_std": 0.1922930534929037, "rewards/accuracy_reward/mean": 0.20982142724096775, "rewards/accuracy_reward/std": 0.37825513631105423, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04423765931278467, "step": 2203 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 804.4576263427734, "completions/mean_terminated_length": 695.9950103759766, "completions/min_length": 261.75, "completions/min_terminated_length": 261.75, "epoch": 0.658352624897319, "grad_norm": 0.22860969603061676, "kl": 2.1171875, "learning_rate": 5.353511282605887e-06, "loss": 0.1155, "num_tokens": 1069510099.0, "reward": 0.6908482611179352, "reward_std": 0.20539475977420807, "rewards/accuracy_reward/mean": 0.21205356949940324, "rewards/accuracy_reward/std": 0.3667946793138981, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04702208936214447, "step": 2204 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 847.3482513427734, "completions/mean_terminated_length": 768.4825286865234, "completions/min_length": 353.75, "completions/min_terminated_length": 353.75, "epoch": 0.6586513329848406, "grad_norm": 0.46504855155944824, "kl": 1.6640625, "learning_rate": 5.343854796748886e-06, "loss": 0.0911, "num_tokens": 1069968831.0, "reward": 0.6657366454601288, "reward_std": 0.13282161764800549, "rewards/accuracy_reward/mean": 0.1741071417927742, "rewards/accuracy_reward/std": 0.3806760907173157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04423765931278467, "step": 2205 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 860.5848541259766, "completions/mean_terminated_length": 765.2259216308594, "completions/min_length": 390.5, "completions/min_terminated_length": 390.5, "epoch": 0.658950041072362, "grad_norm": 0.2566669285297394, "kl": 1.8017578125, "learning_rate": 5.334203851293442e-06, "loss": 0.1049, "num_tokens": 1070429141.0, "reward": 0.6250000298023224, "reward_std": 0.14474719762802124, "rewards/accuracy_reward/mean": 0.13392857322469354, "rewards/accuracy_reward/std": 0.3140723295509815, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.044314838480204344, "step": 2206 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 879.5714721679688, "completions/mean_terminated_length": 778.9593963623047, "completions/min_length": 417.25, "completions/min_terminated_length": 417.25, "epoch": 0.6592487491598835, "grad_norm": 0.2990282475948334, "kl": 2.3359375, "learning_rate": 5.324558457723319e-06, "loss": 0.1188, "num_tokens": 1070896869.0, "reward": 0.6685268133878708, "reward_std": 0.16208236664533615, "rewards/accuracy_reward/mean": 0.18080357182770967, "rewards/accuracy_reward/std": 0.3562675416469574, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 2207 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 808.8616485595703, "completions/mean_terminated_length": 713.0025177001953, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 0.6595474572474049, "grad_norm": 0.4069203734397888, "kl": 2.65625, "learning_rate": 5.31491862751569e-06, "loss": 0.1331, "num_tokens": 1071328215.0, "reward": 0.6835937798023224, "reward_std": 0.19079562835395336, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.3941084146499634, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 2208 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 853.4219207763672, "completions/mean_terminated_length": 743.1436462402344, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "epoch": 0.6598461653349265, "grad_norm": 0.261549711227417, "kl": 1.9814453125, "learning_rate": 5.305284372141095e-06, "loss": 0.097, "num_tokens": 1071782900.0, "reward": 0.6026785969734192, "reward_std": 0.11227495269849896, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.26765289157629013, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03885376825928688, "step": 2209 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 877.5179138183594, "completions/mean_terminated_length": 761.2405853271484, "completions/min_length": 429.0, "completions/min_terminated_length": 429.0, "epoch": 0.6601448734224479, "grad_norm": 0.36316317319869995, "kl": 2.0810546875, "learning_rate": 5.295655703063451e-06, "loss": 0.1118, "num_tokens": 1072248076.0, "reward": 0.563616082072258, "reward_std": 0.11904208920896053, "rewards/accuracy_reward/mean": 0.07366071315482259, "rewards/accuracy_reward/std": 0.2453804798424244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 2210 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 849.0156555175781, "completions/mean_terminated_length": 732.3328399658203, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.6604435815099694, "grad_norm": 0.23218613862991333, "kl": 2.4169921875, "learning_rate": 5.286032631740023e-06, "loss": 0.1302, "num_tokens": 1072699267.0, "reward": 0.6763393133878708, "reward_std": 0.19785145670175552, "rewards/accuracy_reward/mean": 0.1874999962747097, "rewards/accuracy_reward/std": 0.3756711483001709, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 2211 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 855.232177734375, "completions/mean_terminated_length": 725.6455383300781, "completions/min_length": 276.25, "completions/min_terminated_length": 276.25, "epoch": 0.6607422895974908, "grad_norm": 0.3376315236091614, "kl": 1.8828125, "learning_rate": 5.276415169621418e-06, "loss": 0.0905, "num_tokens": 1073152795.0, "reward": 0.702566996216774, "reward_std": 0.16602706909179688, "rewards/accuracy_reward/mean": 0.22581844963133335, "rewards/accuracy_reward/std": 0.37903714925050735, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.038913180120289326, "step": 2212 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.75, "completions/mean_length": 800.1004791259766, "completions/mean_terminated_length": 697.1423187255859, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "epoch": 0.6610409976850123, "grad_norm": 0.3210766315460205, "kl": 2.546875, "learning_rate": 5.2668033281515676e-06, "loss": 0.1443, "num_tokens": 1073582920.0, "reward": 0.6155134290456772, "reward_std": 0.15722377598285675, "rewards/accuracy_reward/mean": 0.12723214365541935, "rewards/accuracy_reward/std": 0.33153094351291656, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.04972758609801531, "step": 2213 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 890.5937957763672, "completions/mean_terminated_length": 770.5351257324219, "completions/min_length": 382.5, "completions/min_terminated_length": 382.5, "epoch": 0.6613397057725338, "grad_norm": 0.2536734938621521, "kl": 1.77734375, "learning_rate": 5.257197118767708e-06, "loss": 0.0961, "num_tokens": 1074057522.0, "reward": 0.7036830633878708, "reward_std": 0.183310154825449, "rewards/accuracy_reward/mean": 0.21205356996506453, "rewards/accuracy_reward/std": 0.3068065792322159, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 2214 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 874.591552734375, "completions/mean_terminated_length": 774.5118255615234, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "epoch": 0.6616384138600553, "grad_norm": 0.17784729599952698, "kl": 1.873046875, "learning_rate": 5.247596552900387e-06, "loss": 0.0981, "num_tokens": 1074524187.0, "reward": 0.6534598469734192, "reward_std": 0.17876292020082474, "rewards/accuracy_reward/mean": 0.16815475653856993, "rewards/accuracy_reward/std": 0.3573690503835678, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983776688576, "step": 2215 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 866.9330749511719, "completions/mean_terminated_length": 762.2713928222656, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "epoch": 0.6619371219475767, "grad_norm": 0.2645540237426758, "kl": 1.662109375, "learning_rate": 5.238001641973422e-06, "loss": 0.0984, "num_tokens": 1074982445.0, "reward": 0.645089328289032, "reward_std": 0.1497308388352394, "rewards/accuracy_reward/mean": 0.15178571548312902, "rewards/accuracy_reward/std": 0.3204656317830086, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 2216 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.25, "completions/mean_length": 850.1808471679688, "completions/mean_terminated_length": 736.9105072021484, "completions/min_length": 314.5, "completions/min_terminated_length": 314.5, "epoch": 0.6622358300350982, "grad_norm": 0.45381221175193787, "kl": 1.7255859375, "learning_rate": 5.228412397403916e-06, "loss": 0.1067, "num_tokens": 1075429534.0, "reward": 0.6210937649011612, "reward_std": 0.1404057890176773, "rewards/accuracy_reward/mean": 0.12946428637951612, "rewards/accuracy_reward/std": 0.3279636576771736, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 2217 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 870.2366485595703, "completions/mean_terminated_length": 779.0637817382812, "completions/min_length": 386.5, "completions/min_terminated_length": 386.5, "epoch": 0.6625345381226196, "grad_norm": 0.24431560933589935, "kl": 1.96484375, "learning_rate": 5.218828830602221e-06, "loss": 0.102, "num_tokens": 1075890376.0, "reward": 0.6250000298023224, "reward_std": 0.1726545374840498, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.33223263919353485, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04605984315276146, "step": 2218 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 876.0178985595703, "completions/mean_terminated_length": 773.2236022949219, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "epoch": 0.6628332462101412, "grad_norm": 0.24738919734954834, "kl": 2.48046875, "learning_rate": 5.2092509529719375e-06, "loss": 0.1323, "num_tokens": 1076346896.0, "reward": 0.7098214477300644, "reward_std": 0.24361436441540718, "rewards/accuracy_reward/mean": 0.2232142835855484, "rewards/accuracy_reward/std": 0.3885052502155304, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 2219 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 881.6116485595703, "completions/mean_terminated_length": 768.7146453857422, "completions/min_length": 438.5, "completions/min_terminated_length": 438.5, "epoch": 0.6631319542976626, "grad_norm": 0.2578669786453247, "kl": 2.603515625, "learning_rate": 5.199678775909889e-06, "loss": 0.1438, "num_tokens": 1076813634.0, "reward": 0.7064732313156128, "reward_std": 0.21072807535529137, "rewards/accuracy_reward/mean": 0.21874999813735485, "rewards/accuracy_reward/std": 0.3904479444026947, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05284425709396601, "step": 2220 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 870.560302734375, "completions/mean_terminated_length": 741.3592376708984, "completions/min_length": 437.0, "completions/min_terminated_length": 437.0, "epoch": 0.6634306623851841, "grad_norm": 0.25520530343055725, "kl": 2.517578125, "learning_rate": 5.190112310806126e-06, "loss": 0.125, "num_tokens": 1077276237.0, "reward": 0.5959821790456772, "reward_std": 0.12652291916310787, "rewards/accuracy_reward/mean": 0.10937500488944352, "rewards/accuracy_reward/std": 0.28724764846265316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05632450245320797, "step": 2221 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 872.1719055175781, "completions/mean_terminated_length": 736.5105743408203, "completions/min_length": 447.5, "completions/min_terminated_length": 447.5, "epoch": 0.6637293704727055, "grad_norm": 0.3556247651576996, "kl": 2.064453125, "learning_rate": 5.1805515690438915e-06, "loss": 0.1288, "num_tokens": 1077732890.0, "reward": 0.7410714477300644, "reward_std": 0.20097294077277184, "rewards/accuracy_reward/mean": 0.2499999962747097, "rewards/accuracy_reward/std": 0.42757244408130646, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 2222 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 870.6719207763672, "completions/mean_terminated_length": 739.5732116699219, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "epoch": 0.664028078560227, "grad_norm": 0.2047225385904312, "kl": 1.939453125, "learning_rate": 5.170996561999631e-06, "loss": 0.0906, "num_tokens": 1078196551.0, "reward": 0.6104910969734192, "reward_std": 0.12467945646494627, "rewards/accuracy_reward/mean": 0.12053571548312902, "rewards/accuracy_reward/std": 0.3051761984825134, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048634594306349754, "step": 2223 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47321428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 868.0112152099609, "completions/mean_terminated_length": 727.990966796875, "completions/min_length": 378.25, "completions/min_terminated_length": 378.25, "epoch": 0.6643267866477485, "grad_norm": 0.3765891194343567, "kl": 1.921875, "learning_rate": 5.161447301042945e-06, "loss": 0.0895, "num_tokens": 1078661932.0, "reward": 0.609933078289032, "reward_std": 0.1888252180069685, "rewards/accuracy_reward/mean": 0.11830357275903225, "rewards/accuracy_reward/std": 0.30382002890110016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 2224 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 907.3817443847656, "completions/mean_terminated_length": 806.7901306152344, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "epoch": 0.66462549473527, "grad_norm": 0.3298906981945038, "kl": 2.234375, "learning_rate": 5.151903797536631e-06, "loss": 0.1238, "num_tokens": 1079141175.0, "reward": 0.6640625298023224, "reward_std": 0.2087509222328663, "rewards/accuracy_reward/mean": 0.17633928637951612, "rewards/accuracy_reward/std": 0.36616238951683044, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 2225 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 921.3638763427734, "completions/mean_terminated_length": 784.0023651123047, "completions/min_length": 479.0, "completions/min_terminated_length": 479.0, "epoch": 0.6649242028227914, "grad_norm": 0.4580206573009491, "kl": 2.34765625, "learning_rate": 5.142366062836599e-06, "loss": 0.1329, "num_tokens": 1079629818.0, "reward": 0.6919643133878708, "reward_std": 0.21510658785700798, "rewards/accuracy_reward/mean": 0.20312500279396772, "rewards/accuracy_reward/std": 0.37340912967920303, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05112028680741787, "step": 2226 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46205357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 860.6339721679688, "completions/mean_terminated_length": 719.5701141357422, "completions/min_length": 257.5, "completions/min_terminated_length": 257.5, "epoch": 0.6652229109103129, "grad_norm": 0.3639717400074005, "kl": 2.84765625, "learning_rate": 5.13283410829192e-06, "loss": 0.1564, "num_tokens": 1080087670.0, "reward": 0.6517857611179352, "reward_std": 0.1673375368118286, "rewards/accuracy_reward/mean": 0.16294642724096775, "rewards/accuracy_reward/std": 0.35178520157933235, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2227 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 897.9219207763672, "completions/mean_terminated_length": 750.8045654296875, "completions/min_length": 367.5, "completions/min_terminated_length": 367.5, "epoch": 0.6655216189978344, "grad_norm": 0.31271377205848694, "kl": 2.75390625, "learning_rate": 5.123307945244772e-06, "loss": 0.145, "num_tokens": 1080564195.0, "reward": 0.6729910969734192, "reward_std": 0.21601391211152077, "rewards/accuracy_reward/mean": 0.18526785634458065, "rewards/accuracy_reward/std": 0.36946453154087067, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05360629688948393, "step": 2228 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5424107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 904.5156707763672, "completions/mean_terminated_length": 765.7596588134766, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.6658203270853559, "grad_norm": 0.7394211292266846, "kl": 3.962890625, "learning_rate": 5.1137875850304545e-06, "loss": 0.196, "num_tokens": 1081050778.0, "reward": 0.659598246216774, "reward_std": 0.22313925996422768, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.37642059475183487, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06091958750039339, "step": 2229 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.0, "completions/mean_length": 886.4330596923828, "completions/mean_terminated_length": 756.3831634521484, "completions/min_length": 405.75, "completions/min_terminated_length": 405.75, "epoch": 0.6661190351728773, "grad_norm": 0.3128589987754822, "kl": 3.3359375, "learning_rate": 5.104273038977346e-06, "loss": 0.1784, "num_tokens": 1081520156.0, "reward": 0.6540178954601288, "reward_std": 0.19047488272190094, "rewards/accuracy_reward/mean": 0.16964285913854837, "rewards/accuracy_reward/std": 0.34366636723279953, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.060001716017723083, "step": 2230 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45758928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 884.8861999511719, "completions/mean_terminated_length": 769.8511199951172, "completions/min_length": 423.75, "completions/min_terminated_length": 423.75, "epoch": 0.6664177432603988, "grad_norm": 0.4797108769416809, "kl": 2.73828125, "learning_rate": 5.094764318406921e-06, "loss": 0.1287, "num_tokens": 1081985369.0, "reward": 0.7204241305589676, "reward_std": 0.17423460260033607, "rewards/accuracy_reward/mean": 0.2343749962747097, "rewards/accuracy_reward/std": 0.4227602034807205, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05715245008468628, "step": 2231 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 880.4598693847656, "completions/mean_terminated_length": 753.5618896484375, "completions/min_length": 374.5, "completions/min_terminated_length": 374.5, "epoch": 0.6667164513479202, "grad_norm": 0.5758154392242432, "kl": 3.6015625, "learning_rate": 5.085261434633717e-06, "loss": 0.1825, "num_tokens": 1082451639.0, "reward": 0.6188616380095482, "reward_std": 0.13409377168864012, "rewards/accuracy_reward/mean": 0.1361607159487903, "rewards/accuracy_reward/std": 0.23921062052249908, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06285234168171883, "step": 2232 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 869.7232666015625, "completions/mean_terminated_length": 721.4381408691406, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "epoch": 0.6670151594354418, "grad_norm": 0.3628866374492645, "kl": 3.68359375, "learning_rate": 5.075764398965331e-06, "loss": 0.201, "num_tokens": 1082917371.0, "reward": 0.6277902126312256, "reward_std": 0.18156875111162663, "rewards/accuracy_reward/mean": 0.1450892835855484, "rewards/accuracy_reward/std": 0.3134915977716446, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008992433548, "rewards/tag_count_reward/std": 0.06086498126387596, "step": 2233 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.6071428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 932.1205749511719, "completions/mean_terminated_length": 793.0162811279297, "completions/min_length": 373.5, "completions/min_terminated_length": 373.5, "epoch": 0.6673138675229632, "grad_norm": 0.35754257440567017, "kl": 3.91015625, "learning_rate": 5.0662732227023866e-06, "loss": 0.1734, "num_tokens": 1083413361.0, "reward": 0.560825914144516, "reward_std": 0.1483510471880436, "rewards/accuracy_reward/mean": 0.08258928637951612, "rewards/accuracy_reward/std": 0.273626372218132, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366082072258, "rewards/tag_count_reward/std": 0.06951384898275137, "step": 2234 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 813.6428985595703, "completions/mean_terminated_length": 693.0367126464844, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "epoch": 0.6676125756104847, "grad_norm": 0.5574373006820679, "kl": 2.640625, "learning_rate": 5.056787917138557e-06, "loss": 0.1402, "num_tokens": 1083849777.0, "reward": 0.7393973618745804, "reward_std": 0.1856373567134142, "rewards/accuracy_reward/mean": 0.254464291036129, "rewards/accuracy_reward/std": 0.43434327840805054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05952764302492142, "step": 2235 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 889.1049499511719, "completions/mean_terminated_length": 771.0941467285156, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "epoch": 0.6679112836980061, "grad_norm": 0.4152793288230896, "kl": 3.32421875, "learning_rate": 5.047308493560506e-06, "loss": 0.1598, "num_tokens": 1084322768.0, "reward": 0.5613839477300644, "reward_std": 0.14158431719988585, "rewards/accuracy_reward/mean": 0.08258928847499192, "rewards/accuracy_reward/std": 0.2076763343065977, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4787946417927742, "rewards/tag_count_reward/std": 0.06968536134809256, "step": 2236 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 851.7991333007812, "completions/mean_terminated_length": 745.652587890625, "completions/min_length": 316.5, "completions/min_terminated_length": 316.5, "epoch": 0.6682099917855276, "grad_norm": 0.2266373485326767, "kl": 3.18359375, "learning_rate": 5.037834963247922e-06, "loss": 0.1541, "num_tokens": 1084772998.0, "reward": 0.659598246216774, "reward_std": 0.1844628443941474, "rewards/accuracy_reward/mean": 0.17633928172290325, "rewards/accuracy_reward/std": 0.3080759719014168, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06127397157251835, "step": 2237 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 881.5446624755859, "completions/mean_terminated_length": 763.0318450927734, "completions/min_length": 362.75, "completions/min_terminated_length": 362.75, "epoch": 0.6685086998730491, "grad_norm": 0.30545955896377563, "kl": 2.90625, "learning_rate": 5.0283673374734546e-06, "loss": 0.1424, "num_tokens": 1085236490.0, "reward": 0.5178571566939354, "reward_std": 0.09391462337225676, "rewards/accuracy_reward/mean": 0.03348214225843549, "rewards/accuracy_reward/std": 0.12768309749662876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.06006421521306038, "step": 2238 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 820.4219055175781, "completions/mean_terminated_length": 713.194091796875, "completions/min_length": 363.75, "completions/min_terminated_length": 363.75, "epoch": 0.6688074079605706, "grad_norm": 0.4572620391845703, "kl": 2.99609375, "learning_rate": 5.0189056275027595e-06, "loss": 0.1873, "num_tokens": 1085672023.0, "reward": 0.6456473618745804, "reward_std": 0.1968010850250721, "rewards/accuracy_reward/mean": 0.16071428637951612, "rewards/accuracy_reward/std": 0.3557771295309067, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05952764302492142, "step": 2239 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 865.6652221679688, "completions/mean_terminated_length": 754.6192932128906, "completions/min_length": 382.75, "completions/min_terminated_length": 382.75, "epoch": 0.669106116048092, "grad_norm": 0.3090040981769562, "kl": 2.341796875, "learning_rate": 5.009449844594425e-06, "loss": 0.1168, "num_tokens": 1086145409.0, "reward": 0.6601562798023224, "reward_std": 0.1646998282521963, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.37351100891828537, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2240 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 840.482177734375, "completions/mean_terminated_length": 729.1726379394531, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "epoch": 0.6694048241356134, "grad_norm": 0.31975191831588745, "kl": 2.96875, "learning_rate": 5.000000000000003e-06, "loss": 0.1548, "num_tokens": 1086597433.0, "reward": 0.643973246216774, "reward_std": 0.179513119161129, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.356784425675869, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05846718233078718, "step": 2241 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 851.8504791259766, "completions/mean_terminated_length": 746.2155303955078, "completions/min_length": 433.75, "completions/min_terminated_length": 433.75, "epoch": 0.669703532223135, "grad_norm": 0.2164681851863861, "kl": 2.3515625, "learning_rate": 4.990556104963967e-06, "loss": 0.1238, "num_tokens": 1087046854.0, "reward": 0.7656250447034836, "reward_std": 0.15491721220314503, "rewards/accuracy_reward/mean": 0.2767857164144516, "rewards/accuracy_reward/std": 0.4432419091463089, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04808079591020942, "step": 2242 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 814.6763763427734, "completions/mean_terminated_length": 732.2437744140625, "completions/min_length": 347.5, "completions/min_terminated_length": 347.5, "epoch": 0.6700022403106564, "grad_norm": 0.3347351849079132, "kl": 2.79296875, "learning_rate": 4.981118170723726e-06, "loss": 0.1585, "num_tokens": 1087484757.0, "reward": 0.688058078289032, "reward_std": 0.17600141186267138, "rewards/accuracy_reward/mean": 0.2087053582072258, "rewards/accuracy_reward/std": 0.3977559953927994, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 2243 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 819.7053985595703, "completions/mean_terminated_length": 723.4059143066406, "completions/min_length": 331.25, "completions/min_terminated_length": 331.25, "epoch": 0.6703009483981779, "grad_norm": 0.21621187031269073, "kl": 1.638671875, "learning_rate": 4.971686208509582e-06, "loss": 0.0833, "num_tokens": 1087920673.0, "reward": 0.5775669813156128, "reward_std": 0.09345208387821913, "rewards/accuracy_reward/mean": 0.08482142770662904, "rewards/accuracy_reward/std": 0.25315575674176216, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04015073226764798, "step": 2244 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34598214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 820.5937957763672, "completions/mean_terminated_length": 709.9204406738281, "completions/min_length": 310.25, "completions/min_terminated_length": 310.25, "epoch": 0.6705996564856993, "grad_norm": 0.2601369619369507, "kl": 2.568359375, "learning_rate": 4.962260229544738e-06, "loss": 0.1406, "num_tokens": 1088363499.0, "reward": 0.7070312947034836, "reward_std": 0.21696611493825912, "rewards/accuracy_reward/mean": 0.22098213993012905, "rewards/accuracy_reward/std": 0.39327918738126755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.057296221144497395, "step": 2245 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39508928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 851.3214569091797, "completions/mean_terminated_length": 734.9640197753906, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.6708983645732208, "grad_norm": 0.26246777176856995, "kl": 1.833984375, "learning_rate": 4.952840245045279e-06, "loss": 0.0926, "num_tokens": 1088821803.0, "reward": 0.6250000149011612, "reward_std": 0.09791925735771656, "rewards/accuracy_reward/mean": 0.13392857182770967, "rewards/accuracy_reward/std": 0.3245723322033882, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 2246 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 835.0580596923828, "completions/mean_terminated_length": 745.1085510253906, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "epoch": 0.6711970726607422, "grad_norm": 0.20091550052165985, "kl": 2.330078125, "learning_rate": 4.943426266220156e-06, "loss": 0.1239, "num_tokens": 1089263637.0, "reward": 0.6277902126312256, "reward_std": 0.15241973102092743, "rewards/accuracy_reward/mean": 0.14062499813735485, "rewards/accuracy_reward/std": 0.33418576419353485, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05148082785308361, "step": 2247 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 847.3527221679688, "completions/mean_terminated_length": 758.7811126708984, "completions/min_length": 377.25, "completions/min_terminated_length": 377.25, "epoch": 0.6714957807482638, "grad_norm": 0.4415164291858673, "kl": 2.458984375, "learning_rate": 4.934018304271167e-06, "loss": 0.132, "num_tokens": 1089719107.0, "reward": 0.7008928954601288, "reward_std": 0.16758779250085354, "rewards/accuracy_reward/mean": 0.2142857126891613, "rewards/accuracy_reward/std": 0.39827995002269745, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05437879078090191, "step": 2248 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33928571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 810.7701263427734, "completions/mean_terminated_length": 704.4083404541016, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.6717944888357852, "grad_norm": 0.2876323163509369, "kl": 2.287109375, "learning_rate": 4.924616370392962e-06, "loss": 0.1373, "num_tokens": 1090152780.0, "reward": 0.6367187798023224, "reward_std": 0.14805346727371216, "rewards/accuracy_reward/mean": 0.14508928451687098, "rewards/accuracy_reward/std": 0.3344468027353287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 2249 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 859.9978179931641, "completions/mean_terminated_length": 760.5966491699219, "completions/min_length": 365.25, "completions/min_terminated_length": 365.25, "epoch": 0.6720931969233067, "grad_norm": 0.234375, "kl": 2.158203125, "learning_rate": 4.915220475773004e-06, "loss": 0.1249, "num_tokens": 1090616171.0, "reward": 0.5937500298023224, "reward_std": 0.14915374107658863, "rewards/accuracy_reward/mean": 0.10267857229337096, "rewards/accuracy_reward/std": 0.28804225474596024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 2250 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 879.1964569091797, "completions/mean_terminated_length": 776.8819885253906, "completions/min_length": 315.25, "completions/min_terminated_length": 315.25, "epoch": 0.6723919050108281, "grad_norm": 0.2308730185031891, "kl": 2.31640625, "learning_rate": 4.9058306315915826e-06, "loss": 0.1271, "num_tokens": 1091083619.0, "reward": 0.6529018133878708, "reward_std": 0.17622903734445572, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.36634204536676407, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04812714271247387, "step": 2251 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.25, "completions/mean_length": 837.1629638671875, "completions/mean_terminated_length": 753.5974426269531, "completions/min_length": 396.25, "completions/min_terminated_length": 396.25, "epoch": 0.6726906130983497, "grad_norm": 0.2779775559902191, "kl": 2.0703125, "learning_rate": 4.896446849021783e-06, "loss": 0.0975, "num_tokens": 1091527564.0, "reward": 0.6205357313156128, "reward_std": 0.10942992870695889, "rewards/accuracy_reward/mean": 0.12946428544819355, "rewards/accuracy_reward/std": 0.2651694566011429, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04444765392690897, "step": 2252 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 858.7969055175781, "completions/mean_terminated_length": 724.4024810791016, "completions/min_length": 317.75, "completions/min_terminated_length": 317.75, "epoch": 0.6729893211858711, "grad_norm": 0.20784255862236023, "kl": 2.22265625, "learning_rate": 4.887069139229481e-06, "loss": 0.1041, "num_tokens": 1091987601.0, "reward": 0.6077009290456772, "reward_std": 0.18135560862720013, "rewards/accuracy_reward/mean": 0.11830357508733869, "rewards/accuracy_reward/std": 0.29003726691007614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 2253 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 866.5402221679688, "completions/mean_terminated_length": 761.7878265380859, "completions/min_length": 368.25, "completions/min_terminated_length": 368.25, "epoch": 0.6732880292733926, "grad_norm": 0.24778366088867188, "kl": 3.01171875, "learning_rate": 4.877697513373315e-06, "loss": 0.1639, "num_tokens": 1092450547.0, "reward": 0.6696428805589676, "reward_std": 0.21877055242657661, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3768375888466835, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843750074505806, "rewards/tag_count_reward/std": 0.06049936171621084, "step": 2254 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 872.7120971679688, "completions/mean_terminated_length": 741.6499633789062, "completions/min_length": 359.75, "completions/min_terminated_length": 359.75, "epoch": 0.673586737360914, "grad_norm": 0.27668431401252747, "kl": 3.09765625, "learning_rate": 4.8683319826047e-06, "loss": 0.1635, "num_tokens": 1092915538.0, "reward": 0.675223246216774, "reward_std": 0.19910700619220734, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3852442353963852, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.0578513452783227, "step": 2255 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 874.9553985595703, "completions/mean_terminated_length": 750.2584991455078, "completions/min_length": 310.25, "completions/min_terminated_length": 310.25, "epoch": 0.6738854454484355, "grad_norm": 0.5171740651130676, "kl": 1.8359375, "learning_rate": 4.858972558067784e-06, "loss": 0.1047, "num_tokens": 1093387726.0, "reward": 0.560267873108387, "reward_std": 0.07579410541802645, "rewards/accuracy_reward/mean": 0.06919642817229033, "rewards/accuracy_reward/std": 0.16111359000205994, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 2256 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 892.4576263427734, "completions/mean_terminated_length": 757.5116882324219, "completions/min_length": 258.5, "completions/min_terminated_length": 258.5, "epoch": 0.674184153535957, "grad_norm": 0.22135595977306366, "kl": 2.091796875, "learning_rate": 4.849619250899458e-06, "loss": 0.0969, "num_tokens": 1093863899.0, "reward": 0.632254496216774, "reward_std": 0.1630343534052372, "rewards/accuracy_reward/mean": 0.14285714109428227, "rewards/accuracy_reward/std": 0.30646139942109585, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 2257 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38169642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 865.3036041259766, "completions/mean_terminated_length": 767.9720306396484, "completions/min_length": 374.25, "completions/min_terminated_length": 374.25, "epoch": 0.6744828616234785, "grad_norm": 0.3660157024860382, "kl": 2.9453125, "learning_rate": 4.840272072229335e-06, "loss": 0.1435, "num_tokens": 1094333395.0, "reward": 0.541294664144516, "reward_std": 0.12086429074406624, "rewards/accuracy_reward/mean": 0.055803569965064526, "rewards/accuracy_reward/std": 0.17534252256155014, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05633127875626087, "step": 2258 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 857.0826263427734, "completions/mean_terminated_length": 743.6594696044922, "completions/min_length": 314.75, "completions/min_terminated_length": 314.75, "epoch": 0.6747815697109999, "grad_norm": 0.48958149552345276, "kl": 1.94140625, "learning_rate": 4.830931033179725e-06, "loss": 0.122, "num_tokens": 1094789160.0, "reward": 0.6612723469734192, "reward_std": 0.17367559298872948, "rewards/accuracy_reward/mean": 0.16964285937137902, "rewards/accuracy_reward/std": 0.3217284493148327, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2259 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3258928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.5, "completions/mean_length": 837.685302734375, "completions/mean_terminated_length": 750.5027618408203, "completions/min_length": 260.0, "completions/min_terminated_length": 260.0, "epoch": 0.6750802777985214, "grad_norm": 0.24059908092021942, "kl": 2.400390625, "learning_rate": 4.821596144865645e-06, "loss": 0.1382, "num_tokens": 1095236395.0, "reward": 0.7098214626312256, "reward_std": 0.14865748398005962, "rewards/accuracy_reward/mean": 0.22098213550634682, "rewards/accuracy_reward/std": 0.3467455245554447, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 2260 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 842.5982360839844, "completions/mean_terminated_length": 742.6486968994141, "completions/min_length": 284.25, "completions/min_terminated_length": 284.25, "epoch": 0.6753789858860428, "grad_norm": 0.2611585855484009, "kl": 1.427734375, "learning_rate": 4.812267418394784e-06, "loss": 0.0924, "num_tokens": 1095682247.0, "reward": 0.6250000298023224, "reward_std": 0.10887458315119147, "rewards/accuracy_reward/mean": 0.12946428474970162, "rewards/accuracy_reward/std": 0.23979217372834682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 2261 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45535714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 876.4643249511719, "completions/mean_terminated_length": 754.5943756103516, "completions/min_length": 299.25, "completions/min_terminated_length": 299.25, "epoch": 0.6756776939735644, "grad_norm": 0.42065635323524475, "kl": 2.9375, "learning_rate": 4.8029448648675094e-06, "loss": 0.146, "num_tokens": 1096141303.0, "reward": 0.6183035969734192, "reward_std": 0.1462414190173149, "rewards/accuracy_reward/mean": 0.12946428544819355, "rewards/accuracy_reward/std": 0.3312687426805496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050612835213541985, "step": 2262 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33928571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 830.0982513427734, "completions/mean_terminated_length": 727.5864715576172, "completions/min_length": 379.0, "completions/min_terminated_length": 379.0, "epoch": 0.6759764020610858, "grad_norm": 0.4579070806503296, "kl": 3.140625, "learning_rate": 4.793628495376826e-06, "loss": 0.1669, "num_tokens": 1096583587.0, "reward": 0.6891741454601288, "reward_std": 0.15323844365775585, "rewards/accuracy_reward/mean": 0.20089285727590322, "rewards/accuracy_reward/std": 0.3533237800002098, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.04530252516269684, "step": 2263 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 870.2098693847656, "completions/mean_terminated_length": 768.3837432861328, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.6762751101486073, "grad_norm": 0.3762539327144623, "kl": 3.15625, "learning_rate": 4.7843183210084025e-06, "loss": 0.1683, "num_tokens": 1097058033.0, "reward": 0.7645089626312256, "reward_std": 0.24124234914779663, "rewards/accuracy_reward/mean": 0.2790178582072258, "rewards/accuracy_reward/std": 0.4433426484465599, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05737193766981363, "step": 2264 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 870.4464569091797, "completions/mean_terminated_length": 761.5956268310547, "completions/min_length": 258.25, "completions/min_terminated_length": 258.25, "epoch": 0.6765738182361287, "grad_norm": 0.4190475344657898, "kl": 2.26171875, "learning_rate": 4.775014352840512e-06, "loss": 0.1224, "num_tokens": 1097517433.0, "reward": 0.7304687649011612, "reward_std": 0.25028908997774124, "rewards/accuracy_reward/mean": 0.2410714253783226, "rewards/accuracy_reward/std": 0.4205450564622879, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04786130925640464, "step": 2265 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.5, "completions/mean_length": 847.7902069091797, "completions/mean_terminated_length": 729.1504516601562, "completions/min_length": 229.5, "completions/min_terminated_length": 229.5, "epoch": 0.6768725263236502, "grad_norm": 0.2500354051589966, "kl": 2.234375, "learning_rate": 4.765716601944062e-06, "loss": 0.1107, "num_tokens": 1097971467.0, "reward": 0.6975446790456772, "reward_std": 0.14036806300282478, "rewards/accuracy_reward/mean": 0.2075892873108387, "rewards/accuracy_reward/std": 0.3987306281924248, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2266 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 863.5469055175781, "completions/mean_terminated_length": 738.7409362792969, "completions/min_length": 283.5, "completions/min_terminated_length": 283.5, "epoch": 0.6771712344111717, "grad_norm": 0.32067838311195374, "kl": 2.59375, "learning_rate": 4.756425079382553e-06, "loss": 0.1282, "num_tokens": 1098429472.0, "reward": 0.6205357536673546, "reward_std": 0.15086411591619253, "rewards/accuracy_reward/mean": 0.14136905036866665, "rewards/accuracy_reward/std": 0.2829900160431862, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051463617011904716, "step": 2267 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 870.8772735595703, "completions/mean_terminated_length": 748.9221649169922, "completions/min_length": 317.75, "completions/min_terminated_length": 317.75, "epoch": 0.6774699424986932, "grad_norm": 0.3129175901412964, "kl": 2.712890625, "learning_rate": 4.74713979621208e-06, "loss": 0.1528, "num_tokens": 1098896361.0, "reward": 0.7237723618745804, "reward_std": 0.20837949588894844, "rewards/accuracy_reward/mean": 0.2366071380674839, "rewards/accuracy_reward/std": 0.3900693617761135, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.055264041759073734, "step": 2268 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5491071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 899.3928985595703, "completions/mean_terminated_length": 757.4590301513672, "completions/min_length": 276.5, "completions/min_terminated_length": 276.5, "epoch": 0.6777686505862146, "grad_norm": 0.3548126518726349, "kl": 1.96875, "learning_rate": 4.7378607634813045e-06, "loss": 0.0943, "num_tokens": 1099370729.0, "reward": 0.6741071790456772, "reward_std": 0.168216984719038, "rewards/accuracy_reward/mean": 0.18303571734577417, "rewards/accuracy_reward/std": 0.3260181248188019, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194977223873, "step": 2269 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 845.7277374267578, "completions/mean_terminated_length": 695.4302978515625, "completions/min_length": 291.75, "completions/min_terminated_length": 291.75, "epoch": 0.6780673586737361, "grad_norm": 0.3654819130897522, "kl": 1.63671875, "learning_rate": 4.728587992231461e-06, "loss": 0.099, "num_tokens": 1099825871.0, "reward": 0.6065848469734192, "reward_std": 0.14910952001810074, "rewards/accuracy_reward/mean": 0.11383928544819355, "rewards/accuracy_reward/std": 0.26120730489492416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 2270 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41071428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 858.9531555175781, "completions/mean_terminated_length": 753.8959503173828, "completions/min_length": 189.0, "completions/min_terminated_length": 189.0, "epoch": 0.6783660667612575, "grad_norm": 0.37940624356269836, "kl": 2.396484375, "learning_rate": 4.7193214934963204e-06, "loss": 0.1402, "num_tokens": 1100283130.0, "reward": 0.6780134066939354, "reward_std": 0.15143634844571352, "rewards/accuracy_reward/mean": 0.1874999962747097, "rewards/accuracy_reward/std": 0.32245247811079025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047717904672026634, "step": 2271 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 883.4152221679688, "completions/mean_terminated_length": 725.2159729003906, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "epoch": 0.6786647748487791, "grad_norm": 0.2535438537597656, "kl": 2.404296875, "learning_rate": 4.710061278302208e-06, "loss": 0.1182, "num_tokens": 1100753492.0, "reward": 0.6155134290456772, "reward_std": 0.13360772654414177, "rewards/accuracy_reward/mean": 0.12723214412108064, "rewards/accuracy_reward/std": 0.30200938135385513, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05163817573338747, "step": 2272 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 886.0937957763672, "completions/mean_terminated_length": 768.0094146728516, "completions/min_length": 402.5, "completions/min_terminated_length": 402.5, "epoch": 0.6789634829363005, "grad_norm": 0.3510034382343292, "kl": 2.416015625, "learning_rate": 4.700807357667953e-06, "loss": 0.1134, "num_tokens": 1101221854.0, "reward": 0.698660746216774, "reward_std": 0.17381735891103745, "rewards/accuracy_reward/mean": 0.21205356740392745, "rewards/accuracy_reward/std": 0.3226903285831213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05581916682422161, "step": 2273 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 839.0982513427734, "completions/mean_terminated_length": 729.2685546875, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "epoch": 0.679262191023822, "grad_norm": 0.3283708095550537, "kl": 2.0, "learning_rate": 4.691559742604906e-06, "loss": 0.1186, "num_tokens": 1101675258.0, "reward": 0.6785714626312256, "reward_std": 0.15250862389802933, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.35491225868463516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 2274 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44419642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 871.3102874755859, "completions/mean_terminated_length": 751.6277770996094, "completions/min_length": 221.0, "completions/min_terminated_length": 221.0, "epoch": 0.6795608991113434, "grad_norm": 0.3544323146343231, "kl": 2.4609375, "learning_rate": 4.682318444116915e-06, "loss": 0.121, "num_tokens": 1102141077.0, "reward": 0.5602678805589676, "reward_std": 0.1483344566076994, "rewards/accuracy_reward/mean": 0.07142857043072581, "rewards/accuracy_reward/std": 0.25003722682595253, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 2275 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 840.0670166015625, "completions/mean_terminated_length": 712.8292999267578, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "epoch": 0.679859607198865, "grad_norm": 0.4287159740924835, "kl": 3.234375, "learning_rate": 4.6730834732003104e-06, "loss": 0.1583, "num_tokens": 1102591331.0, "reward": 0.6049107313156128, "reward_std": 0.13595381937921047, "rewards/accuracy_reward/mean": 0.11830356833525002, "rewards/accuracy_reward/std": 0.28990907594561577, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05471411347389221, "step": 2276 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39955357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 855.7165374755859, "completions/mean_terminated_length": 743.9390258789062, "completions/min_length": 307.25, "completions/min_terminated_length": 307.25, "epoch": 0.6801583152863864, "grad_norm": 0.18234393000602722, "kl": 1.4375, "learning_rate": 4.663854840843885e-06, "loss": 0.0712, "num_tokens": 1103049156.0, "reward": 0.6741071715950966, "reward_std": 0.15756408032029867, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.2894703671336174, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03475248906761408, "step": 2277 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 862.6094055175781, "completions/mean_terminated_length": 738.3423309326172, "completions/min_length": 351.5, "completions/min_terminated_length": 351.5, "epoch": 0.6804570233739079, "grad_norm": 0.4023721516132355, "kl": 1.9296875, "learning_rate": 4.654632558028904e-06, "loss": 0.1153, "num_tokens": 1103502501.0, "reward": 0.718191996216774, "reward_std": 0.17548173293471336, "rewards/accuracy_reward/mean": 0.2276785671710968, "rewards/accuracy_reward/std": 0.4180547446012497, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 2278 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 865.4263763427734, "completions/mean_terminated_length": 732.9676208496094, "completions/min_length": 362.75, "completions/min_terminated_length": 362.75, "epoch": 0.6807557314614293, "grad_norm": 0.3169636130332947, "kl": 2.015625, "learning_rate": 4.645416635729063e-06, "loss": 0.1167, "num_tokens": 1103969412.0, "reward": 0.6121652126312256, "reward_std": 0.15503735281527042, "rewards/accuracy_reward/mean": 0.12276786006987095, "rewards/accuracy_reward/std": 0.32346319407224655, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810296744108, "step": 2279 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 853.4732513427734, "completions/mean_terminated_length": 739.6809234619141, "completions/min_length": 366.5, "completions/min_terminated_length": 366.5, "epoch": 0.6810544395489508, "grad_norm": 0.2973918616771698, "kl": 2.14453125, "learning_rate": 4.636207084910498e-06, "loss": 0.1162, "num_tokens": 1104421288.0, "reward": 0.7176339626312256, "reward_std": 0.22440988942980766, "rewards/accuracy_reward/mean": 0.2276785708963871, "rewards/accuracy_reward/std": 0.39696627855300903, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 2280 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 859.1451263427734, "completions/mean_terminated_length": 749.6836090087891, "completions/min_length": 270.75, "completions/min_terminated_length": 270.75, "epoch": 0.6813531476364723, "grad_norm": 0.3555406630039215, "kl": 2.859375, "learning_rate": 4.627003916531761e-06, "loss": 0.1453, "num_tokens": 1104878441.0, "reward": 0.6936384290456772, "reward_std": 0.22565500810742378, "rewards/accuracy_reward/mean": 0.20758928917348385, "rewards/accuracy_reward/std": 0.3904469758272171, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.05568458419293165, "step": 2281 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 822.5201263427734, "completions/mean_terminated_length": 705.3910675048828, "completions/min_length": 253.0, "completions/min_terminated_length": 253.0, "epoch": 0.6816518557239938, "grad_norm": 0.3376280963420868, "kl": 2.6484375, "learning_rate": 4.617807141543813e-06, "loss": 0.1483, "num_tokens": 1105323826.0, "reward": 0.679129496216774, "reward_std": 0.15120247099548578, "rewards/accuracy_reward/mean": 0.191964291036129, "rewards/accuracy_reward/std": 0.38864271342754364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 2282 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 870.3080749511719, "completions/mean_terminated_length": 744.0967559814453, "completions/min_length": 278.0, "completions/min_terminated_length": 278.0, "epoch": 0.6819505638115152, "grad_norm": 0.3628126084804535, "kl": 1.859375, "learning_rate": 4.608616770889998e-06, "loss": 0.1012, "num_tokens": 1105781996.0, "reward": 0.6975446790456772, "reward_std": 0.16974440403282642, "rewards/accuracy_reward/mean": 0.2075892835855484, "rewards/accuracy_reward/std": 0.4060039520263672, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 2283 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 857.9754791259766, "completions/mean_terminated_length": 740.7209777832031, "completions/min_length": 362.5, "completions/min_terminated_length": 362.5, "epoch": 0.6822492718990366, "grad_norm": 0.21648892760276794, "kl": 2.294921875, "learning_rate": 4.599432815506051e-06, "loss": 0.1222, "num_tokens": 1106227489.0, "reward": 0.6651786118745804, "reward_std": 0.1554933823645115, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.38044489175081253, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051120602525770664, "step": 2284 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5513392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 914.029052734375, "completions/mean_terminated_length": 767.1805419921875, "completions/min_length": 348.75, "completions/min_terminated_length": 348.75, "epoch": 0.6825479799865581, "grad_norm": 0.4141204059123993, "kl": 2.298828125, "learning_rate": 4.590255286320062e-06, "loss": 0.1199, "num_tokens": 1106728478.0, "reward": 0.672433078289032, "reward_std": 0.15122805908322334, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.3847714811563492, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05040315631777048, "step": 2285 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 841.3348693847656, "completions/mean_terminated_length": 758.1448211669922, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.6828466880740796, "grad_norm": 0.241669163107872, "kl": 1.484375, "learning_rate": 4.581084194252486e-06, "loss": 0.0866, "num_tokens": 1107172324.0, "reward": 0.7282366305589676, "reward_std": 0.18848339840769768, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.41520245373249054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03782916208729148, "step": 2286 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 887.216552734375, "completions/mean_terminated_length": 753.5850219726562, "completions/min_length": 384.5, "completions/min_terminated_length": 384.5, "epoch": 0.6831453961616011, "grad_norm": 0.6314462423324585, "kl": 3.91796875, "learning_rate": 4.571919550216107e-06, "loss": 0.1921, "num_tokens": 1107648405.0, "reward": 0.5909598618745804, "reward_std": 0.1920462418347597, "rewards/accuracy_reward/mean": 0.10937500139698386, "rewards/accuracy_reward/std": 0.2802676372230053, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848246216774, "rewards/tag_count_reward/std": 0.06430373527109623, "step": 2287 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 858.5312805175781, "completions/mean_terminated_length": 736.7179107666016, "completions/min_length": 347.75, "completions/min_terminated_length": 347.75, "epoch": 0.6834441042491225, "grad_norm": 0.40210407972335815, "kl": 2.79296875, "learning_rate": 4.5627613651160445e-06, "loss": 0.1354, "num_tokens": 1108110355.0, "reward": 0.6356027126312256, "reward_std": 0.1593025904148817, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.3455350212752819, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 2288 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43303571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 866.7790679931641, "completions/mean_terminated_length": 750.6029052734375, "completions/min_length": 301.25, "completions/min_terminated_length": 301.25, "epoch": 0.683742812336644, "grad_norm": 0.2564166486263275, "kl": 3.349609375, "learning_rate": 4.5536096498497295e-06, "loss": 0.1764, "num_tokens": 1108575168.0, "reward": 0.6584821790456772, "reward_std": 0.17528817430138588, "rewards/accuracy_reward/mean": 0.1741071450524032, "rewards/accuracy_reward/std": 0.3517787903547287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.05946661438792944, "step": 2289 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 830.9777069091797, "completions/mean_terminated_length": 701.7855529785156, "completions/min_length": 248.5, "completions/min_terminated_length": 248.5, "epoch": 0.6840415204241654, "grad_norm": 0.5279719233512878, "kl": 2.712890625, "learning_rate": 4.544464415306898e-06, "loss": 0.1471, "num_tokens": 1109021510.0, "reward": 0.6093750298023224, "reward_std": 0.14137229789048433, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.2723560109734535, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.05471411347389221, "step": 2290 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3303571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 830.3103179931641, "completions/mean_terminated_length": 735.5325164794922, "completions/min_length": 278.25, "completions/min_terminated_length": 278.25, "epoch": 0.684340228511687, "grad_norm": 0.3188469111919403, "kl": 2.423828125, "learning_rate": 4.535325672369567e-06, "loss": 0.1342, "num_tokens": 1109466977.0, "reward": 0.6456473469734192, "reward_std": 0.15895513072609901, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.29810405522584915, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 2291 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 855.5848541259766, "completions/mean_terminated_length": 751.4157257080078, "completions/min_length": 337.75, "completions/min_terminated_length": 337.75, "epoch": 0.6846389365992084, "grad_norm": 0.2729266583919525, "kl": 2.5859375, "learning_rate": 4.526193431912038e-06, "loss": 0.1344, "num_tokens": 1109935783.0, "reward": 0.7092634439468384, "reward_std": 0.20954793691635132, "rewards/accuracy_reward/mean": 0.2209821455180645, "rewards/accuracy_reward/std": 0.38955722004175186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052634578198194504, "step": 2292 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 853.6741333007812, "completions/mean_terminated_length": 738.4328918457031, "completions/min_length": 330.75, "completions/min_terminated_length": 330.75, "epoch": 0.6849376446867299, "grad_norm": 0.198713019490242, "kl": 1.548828125, "learning_rate": 4.517067704800864e-06, "loss": 0.0719, "num_tokens": 1110398133.0, "reward": 0.6875000447034836, "reward_std": 0.19367368891835213, "rewards/accuracy_reward/mean": 0.19419643399305642, "rewards/accuracy_reward/std": 0.34956824593245983, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 2293 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 895.0625457763672, "completions/mean_terminated_length": 776.9383544921875, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "epoch": 0.6852363527742513, "grad_norm": 0.2125815749168396, "kl": 1.763671875, "learning_rate": 4.507948501894857e-06, "loss": 0.0808, "num_tokens": 1110871665.0, "reward": 0.6054687798023224, "reward_std": 0.15837768465280533, "rewards/accuracy_reward/mean": 0.11383928405120969, "rewards/accuracy_reward/std": 0.29740890488028526, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04423765931278467, "step": 2294 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 837.6317291259766, "completions/mean_terminated_length": 711.9015960693359, "completions/min_length": 308.5, "completions/min_terminated_length": 308.5, "epoch": 0.6855350608617728, "grad_norm": 0.25850409269332886, "kl": 3.0078125, "learning_rate": 4.498835834045067e-06, "loss": 0.1751, "num_tokens": 1111318060.0, "reward": 0.7008928954601288, "reward_std": 0.21266254037618637, "rewards/accuracy_reward/mean": 0.21428570849820971, "rewards/accuracy_reward/std": 0.3679163381457329, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 2295 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 845.1629791259766, "completions/mean_terminated_length": 743.4636840820312, "completions/min_length": 356.75, "completions/min_terminated_length": 356.75, "epoch": 0.6858337689492943, "grad_norm": 0.35088449716567993, "kl": 3.58203125, "learning_rate": 4.489729712094762e-06, "loss": 0.1891, "num_tokens": 1111774661.0, "reward": 0.6367187947034836, "reward_std": 0.14711466617882252, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.3485540747642517, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05972688551992178, "step": 2296 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 859.0134124755859, "completions/mean_terminated_length": 765.3917694091797, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 0.6861324770368158, "grad_norm": 0.19080349802970886, "kl": 1.927734375, "learning_rate": 4.480630146879419e-06, "loss": 0.1021, "num_tokens": 1112233099.0, "reward": 0.5959821790456772, "reward_std": 0.12269817106425762, "rewards/accuracy_reward/mean": 0.10491071245633066, "rewards/accuracy_reward/std": 0.2511391956359148, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04605984315276146, "step": 2297 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 869.2879638671875, "completions/mean_terminated_length": 759.3372192382812, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "epoch": 0.6864311851243372, "grad_norm": 0.3438156247138977, "kl": 3.23828125, "learning_rate": 4.471537149226723e-06, "loss": 0.1567, "num_tokens": 1112699676.0, "reward": 0.6774553805589676, "reward_std": 0.2037709355354309, "rewards/accuracy_reward/mean": 0.19196428917348385, "rewards/accuracy_reward/std": 0.38703256100416183, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058666424825787544, "step": 2298 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 996.25, "completions/mean_length": 872.5692443847656, "completions/mean_terminated_length": 745.1956024169922, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "epoch": 0.6867298932118587, "grad_norm": 0.40508556365966797, "kl": 3.47265625, "learning_rate": 4.462450729956531e-06, "loss": 0.167, "num_tokens": 1113163211.0, "reward": 0.636160746216774, "reward_std": 0.16797544807195663, "rewards/accuracy_reward/mean": 0.1517857122235, "rewards/accuracy_reward/std": 0.32476211339235306, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.060200958512723446, "step": 2299 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 836.6027221679688, "completions/mean_terminated_length": 744.7200317382812, "completions/min_length": 389.25, "completions/min_terminated_length": 389.25, "epoch": 0.6870286012993801, "grad_norm": 0.3668493926525116, "kl": 2.7109375, "learning_rate": 4.453370899880888e-06, "loss": 0.1407, "num_tokens": 1113611913.0, "reward": 0.689732164144516, "reward_std": 0.22665449604392052, "rewards/accuracy_reward/mean": 0.20312499813735485, "rewards/accuracy_reward/std": 0.391749732196331, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05643500294536352, "step": 2300 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 852.8482513427734, "completions/mean_terminated_length": 744.5074005126953, "completions/min_length": 283.5, "completions/min_terminated_length": 283.5, "epoch": 0.6873273093869017, "grad_norm": 0.3910664916038513, "kl": 2.7109375, "learning_rate": 4.444297669803981e-06, "loss": 0.1414, "num_tokens": 1114072677.0, "reward": 0.6584821790456772, "reward_std": 0.16876916028559208, "rewards/accuracy_reward/mean": 0.16964286053553224, "rewards/accuracy_reward/std": 0.32907063141465187, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.049176040571182966, "step": 2301 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 849.029052734375, "completions/mean_terminated_length": 712.6055450439453, "completions/min_length": 333.5, "completions/min_terminated_length": 333.5, "epoch": 0.6876260174744231, "grad_norm": 0.5216185450553894, "kl": 2.5625, "learning_rate": 4.435231050522152e-06, "loss": 0.1499, "num_tokens": 1114522626.0, "reward": 0.6512277126312256, "reward_std": 0.1578852590173483, "rewards/accuracy_reward/mean": 0.1618303582072258, "rewards/accuracy_reward/std": 0.34614014625549316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04757413361221552, "step": 2302 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 867.2745819091797, "completions/mean_terminated_length": 740.6654052734375, "completions/min_length": 304.25, "completions/min_terminated_length": 304.25, "epoch": 0.6879247255619446, "grad_norm": 0.2817203104496002, "kl": 2.3125, "learning_rate": 4.42617105282389e-06, "loss": 0.1269, "num_tokens": 1114978285.0, "reward": 0.6612723469734192, "reward_std": 0.2100967951118946, "rewards/accuracy_reward/mean": 0.17187500093132257, "rewards/accuracy_reward/std": 0.3438025191426277, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 2303 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 872.4263763427734, "completions/mean_terminated_length": 772.7222137451172, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "epoch": 0.688223433649466, "grad_norm": 0.4858860373497009, "kl": 2.775390625, "learning_rate": 4.417117687489779e-06, "loss": 0.1417, "num_tokens": 1115445420.0, "reward": 0.5948661118745804, "reward_std": 0.1948009580373764, "rewards/accuracy_reward/mean": 0.10714285867288709, "rewards/accuracy_reward/std": 0.2855798937380314, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05265482235699892, "step": 2304 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 878.5625305175781, "completions/mean_terminated_length": 758.8843536376953, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "epoch": 0.6885221417369876, "grad_norm": 0.20668338239192963, "kl": 1.87890625, "learning_rate": 4.408070965292534e-06, "loss": 0.1057, "num_tokens": 1115911768.0, "reward": 0.792410746216774, "reward_std": 0.2112644724547863, "rewards/accuracy_reward/mean": 0.30133928917348385, "rewards/accuracy_reward/std": 0.44173146039247513, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 2305 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 837.5982513427734, "completions/mean_terminated_length": 731.4387817382812, "completions/min_length": 387.75, "completions/min_terminated_length": 387.75, "epoch": 0.688820849824509, "grad_norm": 0.3017815947532654, "kl": 2.2509765625, "learning_rate": 4.399030896996945e-06, "loss": 0.1173, "num_tokens": 1116360644.0, "reward": 0.6110491305589676, "reward_std": 0.18648213148117065, "rewards/accuracy_reward/mean": 0.1227678544819355, "rewards/accuracy_reward/std": 0.3270955830812454, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050435743760317564, "step": 2306 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 869.0134124755859, "completions/mean_terminated_length": 757.4006500244141, "completions/min_length": 447.0, "completions/min_terminated_length": 447.0, "epoch": 0.6891195579120305, "grad_norm": 0.3370969295501709, "kl": 2.21484375, "learning_rate": 4.389997493359905e-06, "loss": 0.1192, "num_tokens": 1116829850.0, "reward": 0.658482164144516, "reward_std": 0.12359031476080418, "rewards/accuracy_reward/mean": 0.16964285681024194, "rewards/accuracy_reward/std": 0.330112311989069, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 2307 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 860.2232666015625, "completions/mean_terminated_length": 756.5039978027344, "completions/min_length": 376.25, "completions/min_terminated_length": 376.25, "epoch": 0.6894182659995519, "grad_norm": 0.3197386562824249, "kl": 2.2421875, "learning_rate": 4.3809707651303565e-06, "loss": 0.1368, "num_tokens": 1117285598.0, "reward": 0.724888414144516, "reward_std": 0.1996733546257019, "rewards/accuracy_reward/mean": 0.23660714365541935, "rewards/accuracy_reward/std": 0.4144786596298218, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.049277334939688444, "step": 2308 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 850.4107666015625, "completions/mean_terminated_length": 742.3521118164062, "completions/min_length": 293.5, "completions/min_terminated_length": 293.5, "epoch": 0.6897169740870734, "grad_norm": 0.2314574122428894, "kl": 1.92578125, "learning_rate": 4.371950723049314e-06, "loss": 0.1113, "num_tokens": 1117733766.0, "reward": 0.631138414144516, "reward_std": 0.16627062670886517, "rewards/accuracy_reward/mean": 0.14471726305782795, "rewards/accuracy_reward/std": 0.33968110010027885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2309 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 834.8817443847656, "completions/mean_terminated_length": 734.4541473388672, "completions/min_length": 374.5, "completions/min_terminated_length": 374.5, "epoch": 0.6900156821745949, "grad_norm": 0.31348171830177307, "kl": 1.3291015625, "learning_rate": 4.362937377849832e-06, "loss": 0.0775, "num_tokens": 1118180545.0, "reward": 0.701450914144516, "reward_std": 0.15960974991321564, "rewards/accuracy_reward/mean": 0.20758928474970162, "rewards/accuracy_reward/std": 0.35864391550421715, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.037908039055764675, "step": 2310 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 851.3437957763672, "completions/mean_terminated_length": 757.3782653808594, "completions/min_length": 358.25, "completions/min_terminated_length": 358.25, "epoch": 0.6903143902621164, "grad_norm": 0.21339836716651917, "kl": 1.78125, "learning_rate": 4.353930740256997e-06, "loss": 0.0865, "num_tokens": 1118630795.0, "reward": 0.5993303954601288, "reward_std": 0.1185137927532196, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.30892428383231163, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764320462942, "step": 2311 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 856.0089721679688, "completions/mean_terminated_length": 761.0076293945312, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 0.6906130983496378, "grad_norm": 0.22663433849811554, "kl": 1.7734375, "learning_rate": 4.344930820987905e-06, "loss": 0.1037, "num_tokens": 1119076351.0, "reward": 0.6462053805589676, "reward_std": 0.1859026849269867, "rewards/accuracy_reward/mean": 0.15625000186264515, "rewards/accuracy_reward/std": 0.34536947682499886, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764320462942, "step": 2312 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 824.8839874267578, "completions/mean_terminated_length": 742.6310272216797, "completions/min_length": 408.5, "completions/min_terminated_length": 408.5, "epoch": 0.6909118064371593, "grad_norm": 0.2861802279949188, "kl": 2.271484375, "learning_rate": 4.335937630751675e-06, "loss": 0.1348, "num_tokens": 1119520107.0, "reward": 0.718191996216774, "reward_std": 0.17714101448655128, "rewards/accuracy_reward/mean": 0.23214285215362906, "rewards/accuracy_reward/std": 0.3621758297085762, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.055160317569971085, "step": 2313 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 857.8795013427734, "completions/mean_terminated_length": 750.6672058105469, "completions/min_length": 284.25, "completions/min_terminated_length": 284.25, "epoch": 0.6912105145246807, "grad_norm": 0.26812437176704407, "kl": 1.623046875, "learning_rate": 4.326951180249397e-06, "loss": 0.0806, "num_tokens": 1119977493.0, "reward": 0.678013414144516, "reward_std": 0.19329221546649933, "rewards/accuracy_reward/mean": 0.18749999813735485, "rewards/accuracy_reward/std": 0.3613732047379017, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 2314 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 862.7321929931641, "completions/mean_terminated_length": 778.9236755371094, "completions/min_length": 418.75, "completions/min_terminated_length": 418.75, "epoch": 0.6915092226122023, "grad_norm": 0.5615763664245605, "kl": 2.232421875, "learning_rate": 4.317971480174161e-06, "loss": 0.124, "num_tokens": 1120438941.0, "reward": 0.6674107313156128, "reward_std": 0.18087943270802498, "rewards/accuracy_reward/mean": 0.1808035708963871, "rewards/accuracy_reward/std": 0.37508320808410645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05632450245320797, "step": 2315 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 869.9598388671875, "completions/mean_terminated_length": 773.7481231689453, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 0.6918079306997237, "grad_norm": 0.2444220334291458, "kl": 1.5986328125, "learning_rate": 4.308998541211016e-06, "loss": 0.0827, "num_tokens": 1120899963.0, "reward": 0.6953125447034836, "reward_std": 0.1847531609237194, "rewards/accuracy_reward/mean": 0.20312500186264515, "rewards/accuracy_reward/std": 0.3884107246994972, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0410674219019711, "step": 2316 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 865.5178985595703, "completions/mean_terminated_length": 766.0577850341797, "completions/min_length": 399.5, "completions/min_terminated_length": 399.5, "epoch": 0.6921066387872452, "grad_norm": 0.2636839747428894, "kl": 1.45166015625, "learning_rate": 4.30003237403697e-06, "loss": 0.0896, "num_tokens": 1121361635.0, "reward": 0.6277901977300644, "reward_std": 0.19745465368032455, "rewards/accuracy_reward/mean": 0.13616071757860482, "rewards/accuracy_reward/std": 0.30090764723718166, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.037955629639327526, "step": 2317 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 889.1272735595703, "completions/mean_terminated_length": 773.1301116943359, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "epoch": 0.6924053468747666, "grad_norm": 0.44649574160575867, "kl": 1.6875, "learning_rate": 4.291072989320963e-06, "loss": 0.0938, "num_tokens": 1121839260.0, "reward": 0.705357164144516, "reward_std": 0.23065945506095886, "rewards/accuracy_reward/mean": 0.2165178544819355, "rewards/accuracy_reward/std": 0.4107365980744362, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2318 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 872.8482513427734, "completions/mean_terminated_length": 755.426025390625, "completions/min_length": 445.5, "completions/min_terminated_length": 445.5, "epoch": 0.6927040549622882, "grad_norm": 0.37716156244277954, "kl": 1.7724609375, "learning_rate": 4.282120397723879e-06, "loss": 0.1001, "num_tokens": 1122313640.0, "reward": 0.6026786044239998, "reward_std": 0.1417577527463436, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.26516541838645935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03946960438042879, "step": 2319 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 868.0692138671875, "completions/mean_terminated_length": 763.2723693847656, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "epoch": 0.6930027630498096, "grad_norm": 0.1779063642024994, "kl": 1.470703125, "learning_rate": 4.2731746098985035e-06, "loss": 0.0851, "num_tokens": 1122772055.0, "reward": 0.6813616305589676, "reward_std": 0.19492224976420403, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3889601305127144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0420391415245831, "step": 2320 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 878.9219055175781, "completions/mean_terminated_length": 776.9662475585938, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "epoch": 0.6933014711373311, "grad_norm": 0.19595280289649963, "kl": 2.103515625, "learning_rate": 4.264235636489542e-06, "loss": 0.1042, "num_tokens": 1123240212.0, "reward": 0.627232164144516, "reward_std": 0.14806827157735825, "rewards/accuracy_reward/mean": 0.1383928540162742, "rewards/accuracy_reward/std": 0.2909877486526966, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04982579965144396, "step": 2321 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 843.0111846923828, "completions/mean_terminated_length": 727.3894653320312, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.6936001792248525, "grad_norm": 0.3878096640110016, "kl": 2.91015625, "learning_rate": 4.255303488133575e-06, "loss": 0.1462, "num_tokens": 1123687737.0, "reward": 0.5636161044239998, "reward_std": 0.16839869692921638, "rewards/accuracy_reward/mean": 0.0803571455180645, "rewards/accuracy_reward/std": 0.23992109671235085, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.05987984128296375, "step": 2322 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 837.3125457763672, "completions/mean_terminated_length": 713.3362884521484, "completions/min_length": 285.5, "completions/min_terminated_length": 285.5, "epoch": 0.693898887312374, "grad_norm": 0.45012402534484863, "kl": 2.529296875, "learning_rate": 4.246378175459075e-06, "loss": 0.155, "num_tokens": 1124131013.0, "reward": 0.718191996216774, "reward_std": 0.178682466968894, "rewards/accuracy_reward/mean": 0.23214285681024194, "rewards/accuracy_reward/std": 0.36575110629200935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.056536297313869, "step": 2323 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5111607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 900.513427734375, "completions/mean_terminated_length": 779.0591125488281, "completions/min_length": 459.5, "completions/min_terminated_length": 459.5, "epoch": 0.6941975953998955, "grad_norm": 0.3444899320602417, "kl": 2.390625, "learning_rate": 4.2374597090863744e-06, "loss": 0.1296, "num_tokens": 1124616235.0, "reward": 0.6356027126312256, "reward_std": 0.15615327470004559, "rewards/accuracy_reward/mean": 0.1495535671710968, "rewards/accuracy_reward/std": 0.2951769381761551, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.05626536998897791, "step": 2324 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 876.2902221679688, "completions/mean_terminated_length": 773.4094543457031, "completions/min_length": 348.75, "completions/min_terminated_length": 348.75, "epoch": 0.694496303487417, "grad_norm": 0.2381933033466339, "kl": 2.33984375, "learning_rate": 4.228548099627665e-06, "loss": 0.1168, "num_tokens": 1125077597.0, "reward": 0.6015625298023224, "reward_std": 0.1375170201063156, "rewards/accuracy_reward/mean": 0.11383928824216127, "rewards/accuracy_reward/std": 0.29830754920840263, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05285438150167465, "step": 2325 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 845.841552734375, "completions/mean_terminated_length": 731.6547546386719, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.6947950115749384, "grad_norm": 0.31820809841156006, "kl": 2.65234375, "learning_rate": 4.219643357686968e-06, "loss": 0.1413, "num_tokens": 1125528534.0, "reward": 0.7215401977300644, "reward_std": 0.17243488878011703, "rewards/accuracy_reward/mean": 0.2343750037252903, "rewards/accuracy_reward/std": 0.4230629503726959, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05500977020710707, "step": 2326 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 873.4286193847656, "completions/mean_terminated_length": 773.1443328857422, "completions/min_length": 298.5, "completions/min_terminated_length": 298.5, "epoch": 0.6950937196624598, "grad_norm": 0.2472272515296936, "kl": 2.541015625, "learning_rate": 4.210745493860146e-06, "loss": 0.1394, "num_tokens": 1125991334.0, "reward": 0.6852678805589676, "reward_std": 0.17436955496668816, "rewards/accuracy_reward/mean": 0.19642857182770967, "rewards/accuracy_reward/std": 0.3687467500567436, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.049176040571182966, "step": 2327 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46651785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 875.4844055175781, "completions/mean_terminated_length": 748.131591796875, "completions/min_length": 346.5, "completions/min_terminated_length": 346.5, "epoch": 0.6953924277499813, "grad_norm": 0.3356911242008209, "kl": 2.5322265625, "learning_rate": 4.2018545187348645e-06, "loss": 0.1437, "num_tokens": 1126461999.0, "reward": 0.6495536118745804, "reward_std": 0.14460534788668156, "rewards/accuracy_reward/mean": 0.1629464291036129, "rewards/accuracy_reward/std": 0.36822017282247543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.04662088863551617, "step": 2328 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 860.763427734375, "completions/mean_terminated_length": 734.2189025878906, "completions/min_length": 179.0, "completions/min_terminated_length": 179.0, "epoch": 0.6956911358375027, "grad_norm": 0.3776082992553711, "kl": 2.5390625, "learning_rate": 4.192970442890602e-06, "loss": 0.1298, "num_tokens": 1126918837.0, "reward": 0.6015625149011612, "reward_std": 0.14243079535663128, "rewards/accuracy_reward/mean": 0.11904762033373117, "rewards/accuracy_reward/std": 0.3132820576429367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05140746245160699, "step": 2329 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 832.9129943847656, "completions/mean_terminated_length": 730.9872589111328, "completions/min_length": 291.25, "completions/min_terminated_length": 291.25, "epoch": 0.6959898439250243, "grad_norm": 0.7039000391960144, "kl": 3.64453125, "learning_rate": 4.184093276898625e-06, "loss": 0.193, "num_tokens": 1127362926.0, "reward": 0.6841518133878708, "reward_std": 0.19833401963114738, "rewards/accuracy_reward/mean": 0.213913694024086, "rewards/accuracy_reward/std": 0.3988436684012413, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06110059376806021, "step": 2330 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 896.372802734375, "completions/mean_terminated_length": 791.8376770019531, "completions/min_length": 323.25, "completions/min_terminated_length": 323.25, "epoch": 0.6962885520125457, "grad_norm": 0.8399366736412048, "kl": 3.94921875, "learning_rate": 4.17522303132198e-06, "loss": 0.1914, "num_tokens": 1127844549.0, "reward": 0.6713170111179352, "reward_std": 0.20644286647439003, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.37825989350676537, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4815848171710968, "rewards/tag_count_reward/std": 0.06527395732700825, "step": 2331 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 859.2991485595703, "completions/mean_terminated_length": 757.6472015380859, "completions/min_length": 375.5, "completions/min_terminated_length": 375.5, "epoch": 0.6965872601000672, "grad_norm": 0.26093438267707825, "kl": 2.5, "learning_rate": 4.166359716715468e-06, "loss": 0.1305, "num_tokens": 1128300539.0, "reward": 0.6177455633878708, "reward_std": 0.17506450787186623, "rewards/accuracy_reward/mean": 0.12946428544819355, "rewards/accuracy_reward/std": 0.3312687426805496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 2332 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 846.7388763427734, "completions/mean_terminated_length": 733.5793762207031, "completions/min_length": 344.25, "completions/min_terminated_length": 344.25, "epoch": 0.6968859681875886, "grad_norm": 0.7467784881591797, "kl": 2.4482421875, "learning_rate": 4.157503343625659e-06, "loss": 0.1385, "num_tokens": 1128761894.0, "reward": 0.552455373108387, "reward_std": 0.09230300039052963, "rewards/accuracy_reward/mean": 0.0647321417927742, "rewards/accuracy_reward/std": 0.20565498620271683, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05113463895395398, "step": 2333 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 837.9219055175781, "completions/mean_terminated_length": 716.5865936279297, "completions/min_length": 159.25, "completions/min_terminated_length": 159.25, "epoch": 0.6971846762751102, "grad_norm": 0.35029205679893494, "kl": 2.46875, "learning_rate": 4.148653922590845e-06, "loss": 0.1356, "num_tokens": 1129206579.0, "reward": 0.690848246216774, "reward_std": 0.16014419123530388, "rewards/accuracy_reward/mean": 0.20312500186264515, "rewards/accuracy_reward/std": 0.3786919489502907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 2334 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 867.1473693847656, "completions/mean_terminated_length": 758.9454345703125, "completions/min_length": 301.25, "completions/min_terminated_length": 301.25, "epoch": 0.6974833843626316, "grad_norm": 0.19796878099441528, "kl": 2.287109375, "learning_rate": 4.1398114641410655e-06, "loss": 0.1094, "num_tokens": 1129667285.0, "reward": 0.6462053805589676, "reward_std": 0.15772434882819653, "rewards/accuracy_reward/mean": 0.15848214365541935, "rewards/accuracy_reward/std": 0.353863850235939, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.053750067949295044, "step": 2335 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.5, "completions/mean_length": 836.0848693847656, "completions/mean_terminated_length": 722.8348388671875, "completions/min_length": 296.5, "completions/min_terminated_length": 296.5, "epoch": 0.6977820924501531, "grad_norm": 0.5330126881599426, "kl": 2.66796875, "learning_rate": 4.1309759787980565e-06, "loss": 0.1455, "num_tokens": 1130109723.0, "reward": 0.650669664144516, "reward_std": 0.13864553160965443, "rewards/accuracy_reward/mean": 0.1651785671710968, "rewards/accuracy_reward/std": 0.35065434873104095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05607745051383972, "step": 2336 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 837.1183471679688, "completions/mean_terminated_length": 749.9877471923828, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "epoch": 0.6980808005376745, "grad_norm": 0.3094566762447357, "kl": 2.025390625, "learning_rate": 4.12214747707527e-06, "loss": 0.1191, "num_tokens": 1130562464.0, "reward": 0.6635044887661934, "reward_std": 0.10815906524658203, "rewards/accuracy_reward/mean": 0.1741071455180645, "rewards/accuracy_reward/std": 0.30642157047986984, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04884427320212126, "step": 2337 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 859.0804138183594, "completions/mean_terminated_length": 729.3284149169922, "completions/min_length": 284.25, "completions/min_terminated_length": 284.25, "epoch": 0.698379508625196, "grad_norm": 0.41271671652793884, "kl": 2.005859375, "learning_rate": 4.1133259694778415e-06, "loss": 0.0961, "num_tokens": 1131019236.0, "reward": 0.5714285969734192, "reward_std": 0.12966897897422314, "rewards/accuracy_reward/mean": 0.08482143050059676, "rewards/accuracy_reward/std": 0.21966126561164856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05403577908873558, "step": 2338 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43750000000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 863.0893249511719, "completions/mean_terminated_length": 735.7121124267578, "completions/min_length": 225.25, "completions/min_terminated_length": 225.25, "epoch": 0.6986782167127175, "grad_norm": 1.209176778793335, "kl": 2.76953125, "learning_rate": 4.1045114665025905e-06, "loss": 0.1442, "num_tokens": 1131479100.0, "reward": 0.5541294887661934, "reward_std": 0.11457558767870069, "rewards/accuracy_reward/mean": 0.07366071385331452, "rewards/accuracy_reward/std": 0.1989659871906042, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.0577180115506053, "step": 2339 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 824.1406555175781, "completions/mean_terminated_length": 721.5298461914062, "completions/min_length": 355.25, "completions/min_terminated_length": 355.25, "epoch": 0.698976924800239, "grad_norm": 0.3982577323913574, "kl": 2.0712890625, "learning_rate": 4.0957039786379906e-06, "loss": 0.1068, "num_tokens": 1131915979.0, "reward": 0.5859375447034836, "reward_std": 0.11333032883703709, "rewards/accuracy_reward/mean": 0.09598214412108064, "rewards/accuracy_reward/std": 0.274148590862751, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047143861185759306, "step": 2340 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.265625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 799.0491333007812, "completions/mean_terminated_length": 721.1624603271484, "completions/min_length": 266.5, "completions/min_terminated_length": 266.5, "epoch": 0.6992756328877604, "grad_norm": 0.5687451958656311, "kl": 2.79296875, "learning_rate": 4.086903516364179e-06, "loss": 0.1705, "num_tokens": 1132345249.0, "reward": 0.6512276977300644, "reward_std": 0.13057176396250725, "rewards/accuracy_reward/mean": 0.16294642724096775, "rewards/accuracy_reward/std": 0.36486314982175827, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 2341 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33482142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 820.6920013427734, "completions/mean_terminated_length": 718.7469482421875, "completions/min_length": 290.75, "completions/min_terminated_length": 290.75, "epoch": 0.6995743409752819, "grad_norm": 0.23921233415603638, "kl": 2.7265625, "learning_rate": 4.078110090152925e-06, "loss": 0.1563, "num_tokens": 1132782135.0, "reward": 0.602678582072258, "reward_std": 0.17144291661679745, "rewards/accuracy_reward/mean": 0.11607142817229033, "rewards/accuracy_reward/std": 0.2952619343996048, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05409985687583685, "step": 2342 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27901785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 819.5513763427734, "completions/mean_terminated_length": 743.3628234863281, "completions/min_length": 303.0, "completions/min_terminated_length": 303.0, "epoch": 0.6998730490628033, "grad_norm": 0.25299227237701416, "kl": 2.2734375, "learning_rate": 4.06932371046763e-06, "loss": 0.1143, "num_tokens": 1133220014.0, "reward": 0.7260045111179352, "reward_std": 0.1733227912336588, "rewards/accuracy_reward/mean": 0.2343749962747097, "rewards/accuracy_reward/std": 0.42441704124212265, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 2343 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 835.091552734375, "completions/mean_terminated_length": 741.8380279541016, "completions/min_length": 294.25, "completions/min_terminated_length": 294.25, "epoch": 0.7001717571503249, "grad_norm": 0.4825725853443146, "kl": 1.73828125, "learning_rate": 4.06054438776331e-06, "loss": 0.0818, "num_tokens": 1133664919.0, "reward": 0.688058078289032, "reward_std": 0.1668219268321991, "rewards/accuracy_reward/mean": 0.19419642724096775, "rewards/accuracy_reward/std": 0.38588111847639084, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616156578064, "rewards/tag_count_reward/std": 0.03642748761922121, "step": 2344 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 836.6942291259766, "completions/mean_terminated_length": 740.0309295654297, "completions/min_length": 275.75, "completions/min_terminated_length": 275.75, "epoch": 0.7004704652378463, "grad_norm": 0.4808026850223541, "kl": 3.083984375, "learning_rate": 4.051772132486589e-06, "loss": 0.1607, "num_tokens": 1134110478.0, "reward": 0.6640625298023224, "reward_std": 0.18931339122354984, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.36879751831293106, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.051614162512123585, "step": 2345 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 835.2969055175781, "completions/mean_terminated_length": 741.1395721435547, "completions/min_length": 316.75, "completions/min_terminated_length": 316.75, "epoch": 0.7007691733253678, "grad_norm": 0.4653293192386627, "kl": 2.69921875, "learning_rate": 4.043006955075667e-06, "loss": 0.1468, "num_tokens": 1134554451.0, "reward": 0.5758928880095482, "reward_std": 0.11485277768224478, "rewards/accuracy_reward/mean": 0.08705356856808066, "rewards/accuracy_reward/std": 0.20215745642781258, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2346 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34598214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 856.4754791259766, "completions/mean_terminated_length": 769.2146911621094, "completions/min_length": 278.25, "completions/min_terminated_length": 278.25, "epoch": 0.7010678814128892, "grad_norm": 0.46868598461151123, "kl": 3.419921875, "learning_rate": 4.0342488659603354e-06, "loss": 0.1742, "num_tokens": 1135014360.0, "reward": 0.6785714626312256, "reward_std": 0.18628838658332825, "rewards/accuracy_reward/mean": 0.18973213899880648, "rewards/accuracy_reward/std": 0.36328859627246857, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05112028680741787, "step": 2347 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 834.7857513427734, "completions/mean_terminated_length": 750.5206909179688, "completions/min_length": 309.75, "completions/min_terminated_length": 309.75, "epoch": 0.7013665895004108, "grad_norm": 0.3853517174720764, "kl": 3.412109375, "learning_rate": 4.02549787556195e-06, "loss": 0.181, "num_tokens": 1135460104.0, "reward": 0.613839328289032, "reward_std": 0.10565721802413464, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.3336634933948517, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05487643647938967, "step": 2348 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 827.7768096923828, "completions/mean_terminated_length": 724.4964447021484, "completions/min_length": 233.5, "completions/min_terminated_length": 233.5, "epoch": 0.7016652975879322, "grad_norm": 0.43675634264945984, "kl": 2.94921875, "learning_rate": 4.01675399429341e-06, "loss": 0.1496, "num_tokens": 1135902404.0, "reward": 0.7003348618745804, "reward_std": 0.17491945251822472, "rewards/accuracy_reward/mean": 0.2120535708963871, "rewards/accuracy_reward/std": 0.39874494075775146, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052092005498707294, "step": 2349 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 829.7411193847656, "completions/mean_terminated_length": 707.1574249267578, "completions/min_length": 246.25, "completions/min_terminated_length": 246.25, "epoch": 0.7019640056754537, "grad_norm": 0.3584117293357849, "kl": 2.6328125, "learning_rate": 4.008017232559168e-06, "loss": 0.1497, "num_tokens": 1136342448.0, "reward": 0.6506696790456772, "reward_std": 0.10158027336001396, "rewards/accuracy_reward/mean": 0.16071428451687098, "rewards/accuracy_reward/std": 0.3439048305153847, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 2350 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37500000000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 834.013427734375, "completions/mean_terminated_length": 724.9559020996094, "completions/min_length": 355.5, "completions/min_terminated_length": 355.5, "epoch": 0.7022627137629751, "grad_norm": 0.2725412845611572, "kl": 2.33984375, "learning_rate": 3.999287600755192e-06, "loss": 0.1201, "num_tokens": 1136793142.0, "reward": 0.5948660969734192, "reward_std": 0.108853068202734, "rewards/accuracy_reward/mean": 0.10491071362048388, "rewards/accuracy_reward/std": 0.2969711497426033, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 2351 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 784.5848541259766, "completions/mean_terminated_length": 674.2811737060547, "completions/min_length": 281.5, "completions/min_terminated_length": 281.5, "epoch": 0.7025614218504966, "grad_norm": 0.6505053043365479, "kl": 3.03515625, "learning_rate": 3.990565109268977e-06, "loss": 0.1863, "num_tokens": 1137214412.0, "reward": 0.7522321790456772, "reward_std": 0.2144254595041275, "rewards/accuracy_reward/mean": 0.27418154664337635, "rewards/accuracy_reward/std": 0.4262113720178604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 2352 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2700892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 796.0469055175781, "completions/mean_terminated_length": 713.1515350341797, "completions/min_length": 351.5, "completions/min_terminated_length": 351.5, "epoch": 0.702860129938018, "grad_norm": 0.293317049741745, "kl": 2.716796875, "learning_rate": 3.981849768479516e-06, "loss": 0.1503, "num_tokens": 1137637553.0, "reward": 0.6813616454601288, "reward_std": 0.14288048446178436, "rewards/accuracy_reward/mean": 0.1919642835855484, "rewards/accuracy_reward/std": 0.3733600042760372, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.05005982704460621, "step": 2353 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 858.2500457763672, "completions/mean_terminated_length": 736.1016235351562, "completions/min_length": 301.75, "completions/min_terminated_length": 301.75, "epoch": 0.7031588380255396, "grad_norm": 0.2412613183259964, "kl": 1.9140625, "learning_rate": 3.9731415887573e-06, "loss": 0.1133, "num_tokens": 1138100081.0, "reward": 0.6992187649011612, "reward_std": 0.16634758142754436, "rewards/accuracy_reward/mean": 0.20758928917348385, "rewards/accuracy_reward/std": 0.39830178022384644, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 2354 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32366071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 809.7857360839844, "completions/mean_terminated_length": 709.4655914306641, "completions/min_length": 226.75, "completions/min_terminated_length": 226.75, "epoch": 0.703457546113061, "grad_norm": 0.2316247969865799, "kl": 2.98828125, "learning_rate": 3.964440580464286e-06, "loss": 0.1503, "num_tokens": 1138543921.0, "reward": 0.7544643133878708, "reward_std": 0.1434965431690216, "rewards/accuracy_reward/mean": 0.2678571417927742, "rewards/accuracy_reward/std": 0.43619197607040405, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071343421936, "rewards/tag_count_reward/std": 0.055475836619734764, "step": 2355 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 848.6161193847656, "completions/mean_terminated_length": 747.6782531738281, "completions/min_length": 400.75, "completions/min_terminated_length": 400.75, "epoch": 0.7037562542005825, "grad_norm": 0.2730148434638977, "kl": 2.10546875, "learning_rate": 3.955746753953912e-06, "loss": 0.1059, "num_tokens": 1138999045.0, "reward": 0.613839328289032, "reward_std": 0.11332394648343325, "rewards/accuracy_reward/mean": 0.12276785867288709, "rewards/accuracy_reward/std": 0.2975744865834713, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03946960438042879, "step": 2356 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 819.7902221679688, "completions/mean_terminated_length": 730.6558532714844, "completions/min_length": 292.25, "completions/min_terminated_length": 292.25, "epoch": 0.7040549622881039, "grad_norm": 0.41740164160728455, "kl": 2.001953125, "learning_rate": 3.9470601195710575e-06, "loss": 0.0925, "num_tokens": 1139435159.0, "reward": 0.6077009215950966, "reward_std": 0.09214904485270381, "rewards/accuracy_reward/mean": 0.1160714328289032, "rewards/accuracy_reward/std": 0.2566647306084633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2357 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 870.3348541259766, "completions/mean_terminated_length": 746.5815277099609, "completions/min_length": 293.75, "completions/min_terminated_length": 293.75, "epoch": 0.7043536703756255, "grad_norm": 0.22703801095485687, "kl": 1.484375, "learning_rate": 3.938380687652052e-06, "loss": 0.0928, "num_tokens": 1139900829.0, "reward": 0.7154018133878708, "reward_std": 0.19450237229466438, "rewards/accuracy_reward/mean": 0.228050597012043, "rewards/accuracy_reward/std": 0.3909170851111412, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 2358 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 834.6830902099609, "completions/mean_terminated_length": 743.80908203125, "completions/min_length": 250.5, "completions/min_terminated_length": 250.5, "epoch": 0.7046523784631469, "grad_norm": 0.26916375756263733, "kl": 1.4453125, "learning_rate": 3.929708468524655e-06, "loss": 0.0745, "num_tokens": 1140354639.0, "reward": 0.640066996216774, "reward_std": 0.15659330785274506, "rewards/accuracy_reward/mean": 0.1450892835855484, "rewards/accuracy_reward/std": 0.3197513744235039, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 2359 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37276785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 849.3928985595703, "completions/mean_terminated_length": 749.8046112060547, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "epoch": 0.7049510865506684, "grad_norm": 0.21418346464633942, "kl": 1.1953125, "learning_rate": 3.921043472508045e-06, "loss": 0.0663, "num_tokens": 1140806351.0, "reward": 0.7388393133878708, "reward_std": 0.12470057792961597, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.4155838340520859, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 2360 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 898.2076416015625, "completions/mean_terminated_length": 784.9534912109375, "completions/min_length": 332.5, "completions/min_terminated_length": 332.5, "epoch": 0.7052497946381898, "grad_norm": 0.21961070597171783, "kl": 1.771484375, "learning_rate": 3.912385709912794e-06, "loss": 0.0884, "num_tokens": 1141284188.0, "reward": 0.6082589626312256, "reward_std": 0.07621244480833411, "rewards/accuracy_reward/mean": 0.1160714253783226, "rewards/accuracy_reward/std": 0.18390296772122383, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04326625633984804, "step": 2361 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 873.1562957763672, "completions/mean_terminated_length": 774.4182586669922, "completions/min_length": 399.5, "completions/min_terminated_length": 399.5, "epoch": 0.7055485027257113, "grad_norm": 0.31384599208831787, "kl": 2.02734375, "learning_rate": 3.90373519104088e-06, "loss": 0.1087, "num_tokens": 1141745682.0, "reward": 0.636160746216774, "reward_std": 0.15007734671235085, "rewards/accuracy_reward/mean": 0.14732142491266131, "rewards/accuracy_reward/std": 0.3330507129430771, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2362 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 857.6250457763672, "completions/mean_terminated_length": 758.5097351074219, "completions/min_length": 494.75, "completions/min_terminated_length": 494.75, "epoch": 0.7058472108132328, "grad_norm": 0.2986583113670349, "kl": 1.789306640625, "learning_rate": 3.895091926185653e-06, "loss": 0.0923, "num_tokens": 1142202346.0, "reward": 0.6266741305589676, "reward_std": 0.13792114704847336, "rewards/accuracy_reward/mean": 0.13616071455180645, "rewards/accuracy_reward/std": 0.33629459887742996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04098389483988285, "step": 2363 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 902.9018402099609, "completions/mean_terminated_length": 767.0889282226562, "completions/min_length": 355.25, "completions/min_terminated_length": 355.25, "epoch": 0.7061459189007543, "grad_norm": 0.2585299611091614, "kl": 1.88671875, "learning_rate": 3.8864559256318375e-06, "loss": 0.0936, "num_tokens": 1142684510.0, "reward": 0.6071428805589676, "reward_std": 0.15092061460018158, "rewards/accuracy_reward/mean": 0.11607143003493547, "rewards/accuracy_reward/std": 0.30934279412031174, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194977223873, "step": 2364 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 886.3750457763672, "completions/mean_terminated_length": 784.4233856201172, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "epoch": 0.7064446269882757, "grad_norm": 0.23121598362922668, "kl": 1.3623046875, "learning_rate": 3.877827199655506e-06, "loss": 0.0767, "num_tokens": 1143153830.0, "reward": 0.6875000149011612, "reward_std": 0.17458196356892586, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.38130928203463554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.036294152960181236, "step": 2365 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 861.2969207763672, "completions/mean_terminated_length": 758.7824249267578, "completions/min_length": 209.25, "completions/min_terminated_length": 209.25, "epoch": 0.7067433350757972, "grad_norm": 0.1896522343158722, "kl": 1.421875, "learning_rate": 3.869205758524091e-06, "loss": 0.0652, "num_tokens": 1143613611.0, "reward": 0.6835937798023224, "reward_std": 0.1366766393184662, "rewards/accuracy_reward/mean": 0.20014880551025271, "rewards/accuracy_reward/std": 0.36154814064502716, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037521267775446177, "step": 2366 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 868.7768402099609, "completions/mean_terminated_length": 758.085693359375, "completions/min_length": 402.75, "completions/min_terminated_length": 402.75, "epoch": 0.7070420431633186, "grad_norm": 0.2140870839357376, "kl": 2.1103515625, "learning_rate": 3.860591612496335e-06, "loss": 0.1175, "num_tokens": 1144077527.0, "reward": 0.6657366454601288, "reward_std": 0.1655474714934826, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.355879969894886, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.489397332072258, "rewards/tag_count_reward/std": 0.04800507938489318, "step": 2367 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 888.654052734375, "completions/mean_terminated_length": 764.5577392578125, "completions/min_length": 279.5, "completions/min_terminated_length": 279.5, "epoch": 0.7073407512508402, "grad_norm": 0.23793260753154755, "kl": 1.74365234375, "learning_rate": 3.85198477182232e-06, "loss": 0.0929, "num_tokens": 1144546748.0, "reward": 0.593191996216774, "reward_std": 0.14710435643792152, "rewards/accuracy_reward/mean": 0.10267857182770967, "rewards/accuracy_reward/std": 0.293948981910944, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04537561582401395, "step": 2368 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 805.7745971679688, "completions/mean_terminated_length": 691.9066467285156, "completions/min_length": 312.75, "completions/min_terminated_length": 312.75, "epoch": 0.7076394593383616, "grad_norm": 0.328095942735672, "kl": 1.94140625, "learning_rate": 3.8433852467434175e-06, "loss": 0.1076, "num_tokens": 1144972903.0, "reward": 0.678013414144516, "reward_std": 0.15366364922374487, "rewards/accuracy_reward/mean": 0.18750000116415322, "rewards/accuracy_reward/std": 0.33109560422599316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2369 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 880.6652374267578, "completions/mean_terminated_length": 753.7795867919922, "completions/min_length": 279.75, "completions/min_terminated_length": 279.75, "epoch": 0.707938167425883, "grad_norm": 0.2979893982410431, "kl": 1.919921875, "learning_rate": 3.834793047492311e-06, "loss": 0.0887, "num_tokens": 1145442449.0, "reward": 0.585379496216774, "reward_std": 0.12869519460946321, "rewards/accuracy_reward/mean": 0.09598214295692742, "rewards/accuracy_reward/std": 0.2654780987650156, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932655245066, "step": 2370 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 839.4955749511719, "completions/mean_terminated_length": 716.130859375, "completions/min_length": 364.5, "completions/min_terminated_length": 364.5, "epoch": 0.7082368755134045, "grad_norm": 0.3003630042076111, "kl": 2.03125, "learning_rate": 3.826208184292952e-06, "loss": 0.1124, "num_tokens": 1145891535.0, "reward": 0.7154018133878708, "reward_std": 0.21807659789919853, "rewards/accuracy_reward/mean": 0.2321428619325161, "rewards/accuracy_reward/std": 0.4048025980591774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04923219606280327, "step": 2371 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 848.4553985595703, "completions/mean_terminated_length": 730.9932556152344, "completions/min_length": 272.25, "completions/min_terminated_length": 272.25, "epoch": 0.7085355836009259, "grad_norm": 0.3817679286003113, "kl": 1.4755859375, "learning_rate": 3.817630667360573e-06, "loss": 0.0911, "num_tokens": 1146343739.0, "reward": 0.6623884290456772, "reward_std": 0.17548119090497494, "rewards/accuracy_reward/mean": 0.1696428507566452, "rewards/accuracy_reward/std": 0.3668685033917427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2372 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 874.6183471679688, "completions/mean_terminated_length": 770.2476501464844, "completions/min_length": 323.5, "completions/min_terminated_length": 323.5, "epoch": 0.7088342916884475, "grad_norm": 0.8352638483047485, "kl": 1.587890625, "learning_rate": 3.8090605069016596e-06, "loss": 0.0957, "num_tokens": 1146813552.0, "reward": 0.5474330559372902, "reward_std": 0.10100204683840275, "rewards/accuracy_reward/mean": 0.055803571129217744, "rewards/accuracy_reward/std": 0.17239774577319622, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 2373 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 827.9442443847656, "completions/mean_terminated_length": 728.3610992431641, "completions/min_length": 373.25, "completions/min_terminated_length": 373.25, "epoch": 0.7091329997759689, "grad_norm": 0.6113815307617188, "kl": 1.4423828125, "learning_rate": 3.800497713113951e-06, "loss": 0.0813, "num_tokens": 1147255479.0, "reward": 0.685825914144516, "reward_std": 0.09910605195909739, "rewards/accuracy_reward/mean": 0.19419642770662904, "rewards/accuracy_reward/std": 0.2710936479270458, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0420391415245831, "step": 2374 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 827.1116485595703, "completions/mean_terminated_length": 724.8834533691406, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 0.7094317078634904, "grad_norm": 0.2665710747241974, "kl": 1.578125, "learning_rate": 3.7919422961864084e-06, "loss": 0.1082, "num_tokens": 1147693225.0, "reward": 0.6551339477300644, "reward_std": 0.20501606538891792, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.35951562970876694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.034261973574757576, "step": 2375 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 817.0312957763672, "completions/mean_terminated_length": 718.8839874267578, "completions/min_length": 272.0, "completions/min_terminated_length": 272.0, "epoch": 0.7097304159510118, "grad_norm": 0.19746215641498566, "kl": 1.662109375, "learning_rate": 3.7833942662992286e-06, "loss": 0.0887, "num_tokens": 1148134023.0, "reward": 0.6808035969734192, "reward_std": 0.143597312271595, "rewards/accuracy_reward/mean": 0.1874999995343387, "rewards/accuracy_reward/std": 0.3590032309293747, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 2376 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 868.575927734375, "completions/mean_terminated_length": 738.2783203125, "completions/min_length": 391.75, "completions/min_terminated_length": 391.75, "epoch": 0.7100291240385334, "grad_norm": 0.32534679770469666, "kl": 1.5537109375, "learning_rate": 3.774853633623806e-06, "loss": 0.0779, "num_tokens": 1148594665.0, "reward": 0.6021205484867096, "reward_std": 0.12114030495285988, "rewards/accuracy_reward/mean": 0.10974702541716397, "rewards/accuracy_reward/std": 0.2798813786357641, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.028356278780847788, "step": 2377 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 859.2120971679688, "completions/mean_terminated_length": 763.9307556152344, "completions/min_length": 243.75, "completions/min_terminated_length": 243.75, "epoch": 0.7103278321260548, "grad_norm": 0.40105199813842773, "kl": 2.0859375, "learning_rate": 3.7663204083227456e-06, "loss": 0.0949, "num_tokens": 1149052408.0, "reward": 0.7008928954601288, "reward_std": 0.19079787284135818, "rewards/accuracy_reward/mean": 0.2098214216530323, "rewards/accuracy_reward/std": 0.3883897066116333, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 2378 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 906.5893249511719, "completions/mean_terminated_length": 783.49560546875, "completions/min_length": 487.0, "completions/min_terminated_length": 487.0, "epoch": 0.7106265402135763, "grad_norm": 0.26250940561294556, "kl": 2.53515625, "learning_rate": 3.7577946005498224e-06, "loss": 0.133, "num_tokens": 1149531392.0, "reward": 0.5931920036673546, "reward_std": 0.12774765770882368, "rewards/accuracy_reward/mean": 0.10491071129217744, "rewards/accuracy_reward/std": 0.23353252187371254, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05277834925800562, "step": 2379 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 841.1272583007812, "completions/mean_terminated_length": 723.6627502441406, "completions/min_length": 199.0, "completions/min_terminated_length": 199.0, "epoch": 0.7109252483010977, "grad_norm": 0.3301294445991516, "kl": 2.802734375, "learning_rate": 3.7492762204500065e-06, "loss": 0.1483, "num_tokens": 1149978841.0, "reward": 0.6183036118745804, "reward_std": 0.11549297533929348, "rewards/accuracy_reward/mean": 0.12946428474970162, "rewards/accuracy_reward/std": 0.30280483327805996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 2380 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 870.8750457763672, "completions/mean_terminated_length": 726.9387359619141, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "epoch": 0.7112239563886192, "grad_norm": 0.5595300197601318, "kl": 2.62890625, "learning_rate": 3.7407652781594094e-06, "loss": 0.1332, "num_tokens": 1150434417.0, "reward": 0.6322545111179352, "reward_std": 0.1746978759765625, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.3476935774087906, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 2381 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 848.2522583007812, "completions/mean_terminated_length": 752.7858581542969, "completions/min_length": 336.25, "completions/min_terminated_length": 336.25, "epoch": 0.7115226644761407, "grad_norm": 0.3202192187309265, "kl": 1.896484375, "learning_rate": 3.7322617838053066e-06, "loss": 0.109, "num_tokens": 1150883554.0, "reward": 0.5954241305589676, "reward_std": 0.11232558498159051, "rewards/accuracy_reward/mean": 0.10267857275903225, "rewards/accuracy_reward/std": 0.2584189251065254, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 2382 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 885.8705749511719, "completions/mean_terminated_length": 767.3516845703125, "completions/min_length": 341.5, "completions/min_terminated_length": 341.5, "epoch": 0.7118213725636622, "grad_norm": 0.26951199769973755, "kl": 1.74609375, "learning_rate": 3.7237657475060994e-06, "loss": 0.0991, "num_tokens": 1151352392.0, "reward": 0.6467634290456772, "reward_std": 0.137510369066149, "rewards/accuracy_reward/mean": 0.15401786100119352, "rewards/accuracy_reward/std": 0.2792244032025337, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2383 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 868.7143402099609, "completions/mean_terminated_length": 760.7524108886719, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "epoch": 0.7121200806511836, "grad_norm": 0.5353918075561523, "kl": 2.55859375, "learning_rate": 3.715277179371326e-06, "loss": 0.1246, "num_tokens": 1151812056.0, "reward": 0.622209832072258, "reward_std": 0.16710709407925606, "rewards/accuracy_reward/mean": 0.1316964295692742, "rewards/accuracy_reward/std": 0.3121430315077305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04672335181385279, "step": 2384 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 841.3437805175781, "completions/mean_terminated_length": 725.5060729980469, "completions/min_length": 193.75, "completions/min_terminated_length": 193.75, "epoch": 0.7124187887387051, "grad_norm": 0.9613730311393738, "kl": 2.6953125, "learning_rate": 3.7067960895016277e-06, "loss": 0.1482, "num_tokens": 1152265442.0, "reward": 0.6875000447034836, "reward_std": 0.2081766277551651, "rewards/accuracy_reward/mean": 0.19642857275903225, "rewards/accuracy_reward/std": 0.3870370462536812, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714402794838, "rewards/tag_count_reward/std": 0.04480193881317973, "step": 2385 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.0, "completions/mean_length": 843.5670166015625, "completions/mean_terminated_length": 698.4363555908203, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "epoch": 0.7127174968262265, "grad_norm": 0.5598868727684021, "kl": 3.3828125, "learning_rate": 3.698322487988755e-06, "loss": 0.1692, "num_tokens": 1152723472.0, "reward": 0.6662946790456772, "reward_std": 0.16368716210126877, "rewards/accuracy_reward/mean": 0.18080357648432255, "rewards/accuracy_reward/std": 0.36192240566015244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.058194358833134174, "step": 2386 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 849.2946624755859, "completions/mean_terminated_length": 719.8092041015625, "completions/min_length": 332.25, "completions/min_terminated_length": 332.25, "epoch": 0.7130162049137481, "grad_norm": 0.2234545797109604, "kl": 2.224609375, "learning_rate": 3.6898563849155433e-06, "loss": 0.132, "num_tokens": 1153174852.0, "reward": 0.6729911118745804, "reward_std": 0.18374993465840816, "rewards/accuracy_reward/mean": 0.18080357229337096, "rewards/accuracy_reward/std": 0.3347654268145561, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.041961644776165485, "step": 2387 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 825.8705749511719, "completions/mean_terminated_length": 717.738037109375, "completions/min_length": 365.5, "completions/min_terminated_length": 365.5, "epoch": 0.7133149130012695, "grad_norm": 0.26816439628601074, "kl": 2.3671875, "learning_rate": 3.681397790355915e-06, "loss": 0.1425, "num_tokens": 1153615626.0, "reward": 0.681919664144516, "reward_std": 0.1859893761575222, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.39279238134622574, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 2388 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 774.6763763427734, "completions/mean_terminated_length": 666.4685668945312, "completions/min_length": 281.75, "completions/min_terminated_length": 281.75, "epoch": 0.713613621088791, "grad_norm": 0.6737719178199768, "kl": 2.11328125, "learning_rate": 3.67294671437484e-06, "loss": 0.1276, "num_tokens": 1154033305.0, "reward": 0.6690848618745804, "reward_std": 0.15974775329232216, "rewards/accuracy_reward/mean": 0.1834077420644462, "rewards/accuracy_reward/std": 0.35820094496011734, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 2389 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 808.372802734375, "completions/mean_terminated_length": 697.574462890625, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "epoch": 0.7139123291763124, "grad_norm": 0.3784453868865967, "kl": 1.4052734375, "learning_rate": 3.6645031670283616e-06, "loss": 0.0708, "num_tokens": 1154465184.0, "reward": 0.7232143133878708, "reward_std": 0.19377122074365616, "rewards/accuracy_reward/mean": 0.229910708963871, "rewards/accuracy_reward/std": 0.42096415907144547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.493303582072258, "rewards/tag_count_reward/std": 0.03774221893399954, "step": 2390 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 846.2388763427734, "completions/mean_terminated_length": 729.7490539550781, "completions/min_length": 273.5, "completions/min_terminated_length": 273.5, "epoch": 0.714211037263834, "grad_norm": 0.39267995953559875, "kl": 2.666015625, "learning_rate": 3.6560671583635467e-06, "loss": 0.1596, "num_tokens": 1154919019.0, "reward": 0.6651786118745804, "reward_std": 0.18318810686469078, "rewards/accuracy_reward/mean": 0.1785714291036129, "rewards/accuracy_reward/std": 0.37409165501594543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148982971907, "step": 2391 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 847.6272735595703, "completions/mean_terminated_length": 735.4383850097656, "completions/min_length": 282.75, "completions/min_terminated_length": 282.75, "epoch": 0.7145097453513554, "grad_norm": 0.35384389758110046, "kl": 2.26171875, "learning_rate": 3.6476386984185054e-06, "loss": 0.1236, "num_tokens": 1155367972.0, "reward": 0.6623884290456772, "reward_std": 0.16368120536208153, "rewards/accuracy_reward/mean": 0.1741071450524032, "rewards/accuracy_reward/std": 0.3322145529091358, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05243533477187157, "step": 2392 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41517857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 840.7187805175781, "completions/mean_terminated_length": 710.7340545654297, "completions/min_length": 280.75, "completions/min_terminated_length": 280.75, "epoch": 0.7148084534388769, "grad_norm": 0.4099220037460327, "kl": 3.140625, "learning_rate": 3.6392177972223596e-06, "loss": 0.1762, "num_tokens": 1155814134.0, "reward": 0.6227678805589676, "reward_std": 0.1563799325376749, "rewards/accuracy_reward/mean": 0.13839285587891936, "rewards/accuracy_reward/std": 0.29918090254068375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.0604257807135582, "step": 2393 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 876.8348541259766, "completions/mean_terminated_length": 767.0886688232422, "completions/min_length": 416.75, "completions/min_terminated_length": 416.75, "epoch": 0.7151071615263983, "grad_norm": 0.32237401604652405, "kl": 2.4375, "learning_rate": 3.630804464795239e-06, "loss": 0.1235, "num_tokens": 1156276748.0, "reward": 0.6188616454601288, "reward_std": 0.1801983006298542, "rewards/accuracy_reward/mean": 0.13169642630964518, "rewards/accuracy_reward/std": 0.3282386064529419, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.053543152287602425, "step": 2394 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 864.2768249511719, "completions/mean_terminated_length": 750.8064117431641, "completions/min_length": 363.0, "completions/min_terminated_length": 363.0, "epoch": 0.7154058696139198, "grad_norm": 0.2451973259449005, "kl": 1.9404296875, "learning_rate": 3.6223987111482684e-06, "loss": 0.1102, "num_tokens": 1156733288.0, "reward": 0.6573660969734192, "reward_std": 0.19614791870117188, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3618851751089096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.047784130088984966, "step": 2395 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.30580357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 809.1607360839844, "completions/mean_terminated_length": 717.7326202392578, "completions/min_length": 307.75, "completions/min_terminated_length": 307.75, "epoch": 0.7157045777014412, "grad_norm": 0.2765423059463501, "kl": 2.7109375, "learning_rate": 3.614000546283547e-06, "loss": 0.1526, "num_tokens": 1157163248.0, "reward": 0.7327009290456772, "reward_std": 0.2096969336271286, "rewards/accuracy_reward/mean": 0.2455357126891613, "rewards/accuracy_reward/std": 0.4245965927839279, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 2396 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 782.7835235595703, "completions/mean_terminated_length": 701.9586334228516, "completions/min_length": 248.5, "completions/min_terminated_length": 248.5, "epoch": 0.7160032857889628, "grad_norm": 0.35818809270858765, "kl": 2.283203125, "learning_rate": 3.6056099801941535e-06, "loss": 0.1225, "num_tokens": 1157583695.0, "reward": 0.6010044813156128, "reward_std": 0.1520809829235077, "rewards/accuracy_reward/mean": 0.11160714086145163, "rewards/accuracy_reward/std": 0.3090369552373886, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04884427320212126, "step": 2397 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 821.7701263427734, "completions/mean_terminated_length": 728.8304443359375, "completions/min_length": 350.75, "completions/min_terminated_length": 350.75, "epoch": 0.7163019938764842, "grad_norm": 0.28214114904403687, "kl": 1.80859375, "learning_rate": 3.597227022864116e-06, "loss": 0.1129, "num_tokens": 1158029640.0, "reward": 0.6640625298023224, "reward_std": 0.1355944126844406, "rewards/accuracy_reward/mean": 0.17410714086145163, "rewards/accuracy_reward/std": 0.3630257323384285, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04666052386164665, "step": 2398 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 858.6339569091797, "completions/mean_terminated_length": 735.4011840820312, "completions/min_length": 279.25, "completions/min_terminated_length": 279.25, "epoch": 0.7166007019640057, "grad_norm": 0.6264621019363403, "kl": 2.70703125, "learning_rate": 3.5888516842684194e-06, "loss": 0.1378, "num_tokens": 1158483492.0, "reward": 0.6183035969734192, "reward_std": 0.1205773763358593, "rewards/accuracy_reward/mean": 0.13169642817229033, "rewards/accuracy_reward/std": 0.32231224328279495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05451487097889185, "step": 2399 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 855.0290374755859, "completions/mean_terminated_length": 744.6587524414062, "completions/min_length": 259.25, "completions/min_terminated_length": 259.25, "epoch": 0.7168994100515271, "grad_norm": 0.4024392068386078, "kl": 2.216796875, "learning_rate": 3.58048397437297e-06, "loss": 0.1105, "num_tokens": 1158931521.0, "reward": 0.72823666036129, "reward_std": 0.21548115275800228, "rewards/accuracy_reward/mean": 0.2377232164144516, "rewards/accuracy_reward/std": 0.38245338201522827, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 2400 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 864.6808319091797, "completions/mean_terminated_length": 737.8355560302734, "completions/min_length": 286.5, "completions/min_terminated_length": 286.5, "epoch": 0.7171981181390487, "grad_norm": 0.4639376997947693, "kl": 3.18359375, "learning_rate": 3.5721239031346067e-06, "loss": 0.177, "num_tokens": 1159394706.0, "reward": 0.7075893133878708, "reward_std": 0.22529417276382446, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.40005170553922653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.06053628120571375, "step": 2401 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 844.091552734375, "completions/mean_terminated_length": 716.7291564941406, "completions/min_length": 328.0, "completions/min_terminated_length": 328.0, "epoch": 0.7174968262265701, "grad_norm": 0.3899790048599243, "kl": 2.470703125, "learning_rate": 3.563771480501076e-06, "loss": 0.1461, "num_tokens": 1159849931.0, "reward": 0.6936384290456772, "reward_std": 0.21838467195630074, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.4024387151002884, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050607324577867985, "step": 2402 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 853.1428985595703, "completions/mean_terminated_length": 738.5941619873047, "completions/min_length": 306.5, "completions/min_terminated_length": 306.5, "epoch": 0.7177955343140916, "grad_norm": 0.24408505856990814, "kl": 2.390625, "learning_rate": 3.555426716411028e-06, "loss": 0.1169, "num_tokens": 1160307483.0, "reward": 0.7511160969734192, "reward_std": 0.213676605373621, "rewards/accuracy_reward/mean": 0.2633928544819355, "rewards/accuracy_reward/std": 0.43499408662319183, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05375006701797247, "step": 2403 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 868.8437805175781, "completions/mean_terminated_length": 764.8522796630859, "completions/min_length": 376.5, "completions/min_terminated_length": 376.5, "epoch": 0.718094242401613, "grad_norm": 0.21420730650424957, "kl": 2.7734375, "learning_rate": 3.5470896207939853e-06, "loss": 0.1548, "num_tokens": 1160772021.0, "reward": 0.613839328289032, "reward_std": 0.17230645194649696, "rewards/accuracy_reward/mean": 0.12723214272409678, "rewards/accuracy_reward/std": 0.3093763366341591, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05618073232471943, "step": 2404 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 875.5491485595703, "completions/mean_terminated_length": 752.9614410400391, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "epoch": 0.7183929504891345, "grad_norm": 0.3670518398284912, "kl": 3.015625, "learning_rate": 3.5387602035703637e-06, "loss": 0.1662, "num_tokens": 1161233403.0, "reward": 0.6316964626312256, "reward_std": 0.18635503202676773, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.3521457388997078, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.059920444153249264, "step": 2405 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 849.9866485595703, "completions/mean_terminated_length": 746.6633911132812, "completions/min_length": 420.5, "completions/min_terminated_length": 420.5, "epoch": 0.718691658576656, "grad_norm": 0.24804317951202393, "kl": 1.8251953125, "learning_rate": 3.5304384746514273e-06, "loss": 0.095, "num_tokens": 1161688597.0, "reward": 0.6261160969734192, "reward_std": 0.1021438087336719, "rewards/accuracy_reward/mean": 0.13430059771053493, "rewards/accuracy_reward/std": 0.2944635860621929, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2406 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 840.3013763427734, "completions/mean_terminated_length": 725.1197357177734, "completions/min_length": 221.75, "completions/min_terminated_length": 221.75, "epoch": 0.7189903666641775, "grad_norm": 0.22841337323188782, "kl": 2.25, "learning_rate": 3.522124443939302e-06, "loss": 0.1123, "num_tokens": 1162137852.0, "reward": 0.5954241156578064, "reward_std": 0.14228863921016455, "rewards/accuracy_reward/mean": 0.10491071571595967, "rewards/accuracy_reward/std": 0.27747391536831856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681240953505039, "step": 2407 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 835.2210235595703, "completions/mean_terminated_length": 722.5491943359375, "completions/min_length": 316.25, "completions/min_terminated_length": 316.25, "epoch": 0.7192890747516989, "grad_norm": 0.2948901057243347, "kl": 2.35546875, "learning_rate": 3.5138181213269498e-06, "loss": 0.1343, "num_tokens": 1162586847.0, "reward": 0.6785714626312256, "reward_std": 0.18531765788793564, "rewards/accuracy_reward/mean": 0.1897321380674839, "rewards/accuracy_reward/std": 0.37925275415182114, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.0499969981610775, "step": 2408 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 878.6272735595703, "completions/mean_terminated_length": 751.9223785400391, "completions/min_length": 222.75, "completions/min_terminated_length": 222.75, "epoch": 0.7195877828392204, "grad_norm": 0.27746129035949707, "kl": 2.794921875, "learning_rate": 3.505519516698165e-06, "loss": 0.1451, "num_tokens": 1163049240.0, "reward": 0.6835937798023224, "reward_std": 0.22074783593416214, "rewards/accuracy_reward/mean": 0.1986607126891613, "rewards/accuracy_reward/std": 0.33893348276615143, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.06065871845930815, "step": 2409 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 839.2344207763672, "completions/mean_terminated_length": 760.4100341796875, "completions/min_length": 373.0, "completions/min_terminated_length": 373.0, "epoch": 0.7198864909267418, "grad_norm": 0.22670042514801025, "kl": 2.26171875, "learning_rate": 3.4972286399275455e-06, "loss": 0.1138, "num_tokens": 1163497073.0, "reward": 0.6562500298023224, "reward_std": 0.15708625316619873, "rewards/accuracy_reward/mean": 0.16517857694998384, "rewards/accuracy_reward/std": 0.33705899491906166, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 2410 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 843.9085235595703, "completions/mean_terminated_length": 740.5105590820312, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "epoch": 0.7201851990142634, "grad_norm": 0.18550880253314972, "kl": 1.904296875, "learning_rate": 3.4889455008805107e-06, "loss": 0.0981, "num_tokens": 1163945288.0, "reward": 0.6177455633878708, "reward_std": 0.12278286553919315, "rewards/accuracy_reward/mean": 0.12723214458674192, "rewards/accuracy_reward/std": 0.3267563432455063, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 2411 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 852.5469207763672, "completions/mean_terminated_length": 751.2861480712891, "completions/min_length": 321.5, "completions/min_terminated_length": 321.5, "epoch": 0.7204839071017848, "grad_norm": 0.3237314224243164, "kl": 2.580078125, "learning_rate": 3.480670109413258e-06, "loss": 0.1209, "num_tokens": 1164408141.0, "reward": 0.5976562649011612, "reward_std": 0.1417460162192583, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.3076774664223194, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05114053189754486, "step": 2412 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 825.8527221679688, "completions/mean_terminated_length": 712.2090301513672, "completions/min_length": 218.75, "completions/min_terminated_length": 218.75, "epoch": 0.7207826151893062, "grad_norm": 0.3270227015018463, "kl": 2.162109375, "learning_rate": 3.472402475372778e-06, "loss": 0.126, "num_tokens": 1164853643.0, "reward": 0.6529018133878708, "reward_std": 0.1508965566754341, "rewards/accuracy_reward/mean": 0.16294642724096775, "rewards/accuracy_reward/std": 0.3603968694806099, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688959056511521, "step": 2413 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 824.6897735595703, "completions/mean_terminated_length": 723.4152374267578, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "epoch": 0.7210813232768277, "grad_norm": 0.19320982694625854, "kl": 1.69140625, "learning_rate": 3.46414260859682e-06, "loss": 0.0918, "num_tokens": 1165290176.0, "reward": 0.7181919813156128, "reward_std": 0.13632692024111748, "rewards/accuracy_reward/mean": 0.2254464328289032, "rewards/accuracy_reward/std": 0.4070488288998604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2414 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 827.0357513427734, "completions/mean_terminated_length": 729.1390991210938, "completions/min_length": 290.5, "completions/min_terminated_length": 290.5, "epoch": 0.7213800313643491, "grad_norm": 0.22009946405887604, "kl": 2.3671875, "learning_rate": 3.455890518913897e-06, "loss": 0.1191, "num_tokens": 1165732096.0, "reward": 0.7137277275323868, "reward_std": 0.24654921516776085, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.4073769897222519, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052634578198194504, "step": 2415 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 846.0491485595703, "completions/mean_terminated_length": 748.5222930908203, "completions/min_length": 341.5, "completions/min_terminated_length": 341.5, "epoch": 0.7216787394518707, "grad_norm": 0.2414143830537796, "kl": 2.2333984375, "learning_rate": 3.4476462161432678e-06, "loss": 0.1228, "num_tokens": 1166177846.0, "reward": 0.6473214477300644, "reward_std": 0.15999888442456722, "rewards/accuracy_reward/mean": 0.15662202215753496, "rewards/accuracy_reward/std": 0.28479254618287086, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.03992343507707119, "step": 2416 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 871.529052734375, "completions/mean_terminated_length": 738.6565399169922, "completions/min_length": 270.75, "completions/min_terminated_length": 270.75, "epoch": 0.7219774475393921, "grad_norm": 0.21627859771251678, "kl": 2.2734375, "learning_rate": 3.4394097100949286e-06, "loss": 0.102, "num_tokens": 1166643555.0, "reward": 0.6763393133878708, "reward_std": 0.1315653882920742, "rewards/accuracy_reward/mean": 0.18749999813735485, "rewards/accuracy_reward/std": 0.38338184356689453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 2417 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37946428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 846.5536041259766, "completions/mean_terminated_length": 740.7866973876953, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.7222761556269136, "grad_norm": 0.23776692152023315, "kl": 2.361328125, "learning_rate": 3.4311810105695875e-06, "loss": 0.1192, "num_tokens": 1167092203.0, "reward": 0.6255580559372902, "reward_std": 0.11835226230323315, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.2871919199824333, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 2418 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 841.3638763427734, "completions/mean_terminated_length": 729.5699310302734, "completions/min_length": 344.25, "completions/min_terminated_length": 344.25, "epoch": 0.722574863714435, "grad_norm": 0.4576314091682434, "kl": 2.232421875, "learning_rate": 3.4229601273586757e-06, "loss": 0.1233, "num_tokens": 1167547806.0, "reward": 0.6741071939468384, "reward_std": 0.15793761797249317, "rewards/accuracy_reward/mean": 0.1990327350795269, "rewards/accuracy_reward/std": 0.37352943792939186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 2419 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 835.9553985595703, "completions/mean_terminated_length": 729.5470733642578, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "epoch": 0.7228735718019565, "grad_norm": 0.44267401099205017, "kl": 2.185546875, "learning_rate": 3.414747070244312e-06, "loss": 0.1221, "num_tokens": 1167995786.0, "reward": 0.7237723618745804, "reward_std": 0.15500598773360252, "rewards/accuracy_reward/mean": 0.23214286006987095, "rewards/accuracy_reward/std": 0.3861864432692528, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 2420 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 826.9911193847656, "completions/mean_terminated_length": 733.6332702636719, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "epoch": 0.723172279889478, "grad_norm": 0.2951214611530304, "kl": 2.3671875, "learning_rate": 3.4065418489993118e-06, "loss": 0.1355, "num_tokens": 1168437174.0, "reward": 0.6674107611179352, "reward_std": 0.18161884136497974, "rewards/accuracy_reward/mean": 0.18229166604578495, "rewards/accuracy_reward/std": 0.36934635788202286, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.0487851407378912, "step": 2421 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 866.4687957763672, "completions/mean_terminated_length": 767.6797943115234, "completions/min_length": 389.0, "completions/min_terminated_length": 389.0, "epoch": 0.7234709879769995, "grad_norm": 0.34522274136543274, "kl": 2.265625, "learning_rate": 3.398344473387165e-06, "loss": 0.106, "num_tokens": 1168904184.0, "reward": 0.5563616454601288, "reward_std": 0.13672815449535847, "rewards/accuracy_reward/mean": 0.06696428777649999, "rewards/accuracy_reward/std": 0.23581060394644737, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 2422 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35937500000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 837.8080596923828, "completions/mean_terminated_length": 733.8800354003906, "completions/min_length": 382.5, "completions/min_terminated_length": 382.5, "epoch": 0.7237696960645209, "grad_norm": 0.20259949564933777, "kl": 2.2734375, "learning_rate": 3.390154953162026e-06, "loss": 0.1125, "num_tokens": 1169353634.0, "reward": 0.647879496216774, "reward_std": 0.1712638884782791, "rewards/accuracy_reward/mean": 0.1584821380674839, "rewards/accuracy_reward/std": 0.3650749921798706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04955237451940775, "step": 2423 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 840.3549499511719, "completions/mean_terminated_length": 736.6362457275391, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "epoch": 0.7240684041520424, "grad_norm": 0.36443769931793213, "kl": 2.560546875, "learning_rate": 3.381973298068696e-06, "loss": 0.1395, "num_tokens": 1169813121.0, "reward": 0.6757812798023224, "reward_std": 0.21001321636140347, "rewards/accuracy_reward/mean": 0.18749999441206455, "rewards/accuracy_reward/std": 0.3585918955504894, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05263457726687193, "step": 2424 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 839.9107513427734, "completions/mean_terminated_length": 739.1094055175781, "completions/min_length": 272.5, "completions/min_terminated_length": 272.5, "epoch": 0.7243671122395638, "grad_norm": 0.2658259868621826, "kl": 2.2958984375, "learning_rate": 3.3737995178426276e-06, "loss": 0.126, "num_tokens": 1170260041.0, "reward": 0.6467634290456772, "reward_std": 0.13992025144398212, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.3589046224951744, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.039679599925875664, "step": 2425 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 869.3504943847656, "completions/mean_terminated_length": 779.6641387939453, "completions/min_length": 423.5, "completions/min_terminated_length": 423.5, "epoch": 0.7246658203270854, "grad_norm": 0.17843714356422424, "kl": 1.4189453125, "learning_rate": 3.3656336222098907e-06, "loss": 0.0794, "num_tokens": 1170724214.0, "reward": 0.6897321790456772, "reward_std": 0.1403541136533022, "rewards/accuracy_reward/mean": 0.2000000039115548, "rewards/accuracy_reward/std": 0.3631575033068657, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 2426 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35044642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 842.3906555175781, "completions/mean_terminated_length": 748.9581451416016, "completions/min_length": 320.0, "completions/min_terminated_length": 320.0, "epoch": 0.7249645284146068, "grad_norm": 0.23758184909820557, "kl": 2.03515625, "learning_rate": 3.3574756208871862e-06, "loss": 0.1105, "num_tokens": 1171177877.0, "reward": 0.6250000298023224, "reward_std": 0.1659994199872017, "rewards/accuracy_reward/mean": 0.1339285676367581, "rewards/accuracy_reward/std": 0.31397004798054695, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 2427 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 834.1719055175781, "completions/mean_terminated_length": 748.1930236816406, "completions/min_length": 311.25, "completions/min_terminated_length": 311.25, "epoch": 0.7252632365021283, "grad_norm": 0.2297985553741455, "kl": 1.59765625, "learning_rate": 3.349325523581809e-06, "loss": 0.0761, "num_tokens": 1171626306.0, "reward": 0.7879464775323868, "reward_std": 0.17048651538789272, "rewards/accuracy_reward/mean": 0.29464286006987095, "rewards/accuracy_reward/std": 0.4093499183654785, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2428 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 848.154052734375, "completions/mean_terminated_length": 727.0681457519531, "completions/min_length": 272.25, "completions/min_terminated_length": 272.25, "epoch": 0.7255619445896497, "grad_norm": 0.24470221996307373, "kl": 2.109375, "learning_rate": 3.3411833399916584e-06, "loss": 0.1081, "num_tokens": 1172075991.0, "reward": 0.5396205633878708, "reward_std": 0.12383725121617317, "rewards/accuracy_reward/mean": 0.04910714365541935, "rewards/accuracy_reward/std": 0.18480264022946358, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 2429 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 831.6942138671875, "completions/mean_terminated_length": 741.7314910888672, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "epoch": 0.7258606526771713, "grad_norm": 0.3817644417285919, "kl": 2.5947265625, "learning_rate": 3.3330490798052128e-06, "loss": 0.1251, "num_tokens": 1172525086.0, "reward": 0.7059151977300644, "reward_std": 0.13630246929824352, "rewards/accuracy_reward/mean": 0.2165178619325161, "rewards/accuracy_reward/std": 0.4019850715994835, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.489397332072258, "rewards/tag_count_reward/std": 0.04800507938489318, "step": 2430 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 835.7969207763672, "completions/mean_terminated_length": 732.6594390869141, "completions/min_length": 229.25, "completions/min_terminated_length": 229.25, "epoch": 0.7261593607646927, "grad_norm": 0.2703322470188141, "kl": 1.55078125, "learning_rate": 3.324922752701528e-06, "loss": 0.0926, "num_tokens": 1172978307.0, "reward": 0.7438616454601288, "reward_std": 0.23157616704702377, "rewards/accuracy_reward/mean": 0.2499999925494194, "rewards/accuracy_reward/std": 0.42637090384960175, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 2431 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 867.7857666015625, "completions/mean_terminated_length": 766.3971710205078, "completions/min_length": 368.75, "completions/min_terminated_length": 368.75, "epoch": 0.7264580688522142, "grad_norm": 0.3502497971057892, "kl": 1.9375, "learning_rate": 3.3168043683502082e-06, "loss": 0.1117, "num_tokens": 1173443283.0, "reward": 0.5976562649011612, "reward_std": 0.14251277409493923, "rewards/accuracy_reward/mean": 0.10639880923554301, "rewards/accuracy_reward/std": 0.2526182755827904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04015073226764798, "step": 2432 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 862.4777069091797, "completions/mean_terminated_length": 763.1198577880859, "completions/min_length": 436.75, "completions/min_terminated_length": 436.75, "epoch": 0.7267567769397356, "grad_norm": 0.21865960955619812, "kl": 1.72265625, "learning_rate": 3.308693936411421e-06, "loss": 0.0918, "num_tokens": 1173898473.0, "reward": 0.6690848469734192, "reward_std": 0.19950556382536888, "rewards/accuracy_reward/mean": 0.17633928102441132, "rewards/accuracy_reward/std": 0.34001884423196316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.0412445142865181, "step": 2433 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 832.2344207763672, "completions/mean_terminated_length": 745.3926849365234, "completions/min_length": 414.25, "completions/min_terminated_length": 414.25, "epoch": 0.7270554850272571, "grad_norm": 0.502835214138031, "kl": 1.8056640625, "learning_rate": 3.3005914665358563e-06, "loss": 0.0852, "num_tokens": 1174343426.0, "reward": 0.7739955633878708, "reward_std": 0.23000287637114525, "rewards/accuracy_reward/mean": 0.2916666604578495, "rewards/accuracy_reward/std": 0.4511248245835304, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040314854588359594, "step": 2434 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 841.0022735595703, "completions/mean_terminated_length": 749.8344879150391, "completions/min_length": 265.75, "completions/min_terminated_length": 265.75, "epoch": 0.7273541931147786, "grad_norm": 0.29380714893341064, "kl": 1.615234375, "learning_rate": 3.2924969683647424e-06, "loss": 0.0749, "num_tokens": 1174794835.0, "reward": 0.6863839626312256, "reward_std": 0.17676964029669762, "rewards/accuracy_reward/mean": 0.1919642831198871, "rewards/accuracy_reward/std": 0.3653133437037468, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 2435 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 915.2879943847656, "completions/mean_terminated_length": 806.386962890625, "completions/min_length": 374.25, "completions/min_terminated_length": 374.25, "epoch": 0.7276529012023001, "grad_norm": 0.28610503673553467, "kl": 2.2021484375, "learning_rate": 3.284410451529816e-06, "loss": 0.1164, "num_tokens": 1175274980.0, "reward": 0.6623884290456772, "reward_std": 0.19351718691177666, "rewards/accuracy_reward/mean": 0.1718749962747097, "rewards/accuracy_reward/std": 0.3087756335735321, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04517605667933822, "step": 2436 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 887.1094207763672, "completions/mean_terminated_length": 778.4502868652344, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.7279516092898215, "grad_norm": 0.36231353878974915, "kl": 1.73046875, "learning_rate": 3.2763319256533177e-06, "loss": 0.0868, "num_tokens": 1175743765.0, "reward": 0.6350446790456772, "reward_std": 0.13775473460555077, "rewards/accuracy_reward/mean": 0.1476934514939785, "rewards/accuracy_reward/std": 0.35433942824602127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 2437 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.0, "completions/mean_length": 833.1339721679688, "completions/mean_terminated_length": 747.3306732177734, "completions/min_length": 368.75, "completions/min_terminated_length": 368.75, "epoch": 0.728250317377343, "grad_norm": 0.3321162462234497, "kl": 1.7939453125, "learning_rate": 3.268261400347984e-06, "loss": 0.1054, "num_tokens": 1176190625.0, "reward": 0.6216518133878708, "reward_std": 0.1724868156015873, "rewards/accuracy_reward/mean": 0.12946428824216127, "rewards/accuracy_reward/std": 0.3073797933757305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04182914597913623, "step": 2438 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 828.2924499511719, "completions/mean_terminated_length": 741.4713134765625, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 0.7285490254648644, "grad_norm": 0.2533194422721863, "kl": 1.818359375, "learning_rate": 3.2601988852170207e-06, "loss": 0.0929, "num_tokens": 1176634948.0, "reward": 0.6071428954601288, "reward_std": 0.10315721668303013, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.3035091161727905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 2439 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 830.3214569091797, "completions/mean_terminated_length": 712.1925201416016, "completions/min_length": 365.0, "completions/min_terminated_length": 365.0, "epoch": 0.728847733552386, "grad_norm": 0.31876057386398315, "kl": 1.40625, "learning_rate": 3.252144389854115e-06, "loss": 0.0879, "num_tokens": 1177075268.0, "reward": 0.7293527126312256, "reward_std": 0.18600307032465935, "rewards/accuracy_reward/mean": 0.23437500558793545, "rewards/accuracy_reward/std": 0.39833731949329376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.029593830928206444, "step": 2440 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 887.6563110351562, "completions/mean_terminated_length": 745.2649078369141, "completions/min_length": 470.75, "completions/min_terminated_length": 470.75, "epoch": 0.7291464416399074, "grad_norm": 0.4604977071285248, "kl": 2.326171875, "learning_rate": 3.2440979238433977e-06, "loss": 0.1166, "num_tokens": 1177550554.0, "reward": 0.616629496216774, "reward_std": 0.11504662781953812, "rewards/accuracy_reward/mean": 0.12723214668221772, "rewards/accuracy_reward/std": 0.23182547464966774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932655245066, "step": 2441 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45535714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 891.4330749511719, "completions/mean_terminated_length": 781.1653900146484, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "epoch": 0.7294451497274289, "grad_norm": 0.6118922233581543, "kl": 2.841796875, "learning_rate": 3.2360594967594606e-06, "loss": 0.1292, "num_tokens": 1178026300.0, "reward": 0.7003348618745804, "reward_std": 0.18321901187300682, "rewards/accuracy_reward/mean": 0.21205357741564512, "rewards/accuracy_reward/std": 0.37923093885183334, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050435743760317564, "step": 2442 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 866.2879791259766, "completions/mean_terminated_length": 738.1274719238281, "completions/min_length": 289.25, "completions/min_terminated_length": 289.25, "epoch": 0.7297438578149503, "grad_norm": 0.3700544238090515, "kl": 2.107421875, "learning_rate": 3.228029118167311e-06, "loss": 0.1198, "num_tokens": 1178492269.0, "reward": 0.6640625298023224, "reward_std": 0.1981833130121231, "rewards/accuracy_reward/mean": 0.17410714086145163, "rewards/accuracy_reward/std": 0.34172311425209045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046347017865628004, "step": 2443 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 855.8460235595703, "completions/mean_terminated_length": 742.6739349365234, "completions/min_length": 330.75, "completions/min_terminated_length": 330.75, "epoch": 0.7300425659024719, "grad_norm": 0.46916428208351135, "kl": 2.3828125, "learning_rate": 3.2200067976224037e-06, "loss": 0.1136, "num_tokens": 1178948344.0, "reward": 0.5758928805589676, "reward_std": 0.15334277972579002, "rewards/accuracy_reward/mean": 0.08928571571595967, "rewards/accuracy_reward/std": 0.25471435487270355, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05417029373347759, "step": 2444 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 871.5937805175781, "completions/mean_terminated_length": 756.8858642578125, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "epoch": 0.7303412739899933, "grad_norm": 0.2457149624824524, "kl": 1.671875, "learning_rate": 3.2119925446705824e-06, "loss": 0.0916, "num_tokens": 1179408930.0, "reward": 0.6679687649011612, "reward_std": 0.1239989809691906, "rewards/accuracy_reward/mean": 0.17633928451687098, "rewards/accuracy_reward/std": 0.3588093891739845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043343435507267714, "step": 2445 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 853.7053833007812, "completions/mean_terminated_length": 728.6766510009766, "completions/min_length": 245.25, "completions/min_terminated_length": 245.25, "epoch": 0.7306399820775148, "grad_norm": 0.45738619565963745, "kl": 1.912109375, "learning_rate": 3.2039863688481055e-06, "loss": 0.0841, "num_tokens": 1179863566.0, "reward": 0.6372768133878708, "reward_std": 0.1044555869884789, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.26461321115493774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764227330685, "step": 2446 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.75, "completions/mean_length": 890.8571624755859, "completions/mean_terminated_length": 761.725830078125, "completions/min_length": 394.75, "completions/min_terminated_length": 394.75, "epoch": 0.7309386901650362, "grad_norm": 0.19147509336471558, "kl": 1.8916015625, "learning_rate": 3.195988279681609e-06, "loss": 0.09, "num_tokens": 1180336318.0, "reward": 0.5976562798023224, "reward_std": 0.11820540390908718, "rewards/accuracy_reward/mean": 0.10751488315872848, "rewards/accuracy_reward/std": 0.28162064775824547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04537529917433858, "step": 2447 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 796.4777069091797, "completions/mean_terminated_length": 700.2930603027344, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "epoch": 0.7312373982525577, "grad_norm": 0.3651692569255829, "kl": 2.5908203125, "learning_rate": 3.18799828668812e-06, "loss": 0.1458, "num_tokens": 1180758900.0, "reward": 0.7678571790456772, "reward_std": 0.1895680371671915, "rewards/accuracy_reward/mean": 0.2812500037252903, "rewards/accuracy_reward/std": 0.4482487589120865, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05437879264354706, "step": 2448 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42633928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 857.6116333007812, "completions/mean_terminated_length": 741.032958984375, "completions/min_length": 401.75, "completions/min_terminated_length": 401.75, "epoch": 0.7315361063400792, "grad_norm": 0.3906717896461487, "kl": 2.16796875, "learning_rate": 3.1800163993750166e-06, "loss": 0.1259, "num_tokens": 1181212454.0, "reward": 0.736607164144516, "reward_std": 0.21204298920929432, "rewards/accuracy_reward/mean": 0.2500000046566129, "rewards/accuracy_reward/std": 0.38269228488206863, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05618073232471943, "step": 2449 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 839.4911193847656, "completions/mean_terminated_length": 742.2701263427734, "completions/min_length": 358.5, "completions/min_terminated_length": 358.5, "epoch": 0.7318348144276007, "grad_norm": 0.30131620168685913, "kl": 1.58740234375, "learning_rate": 3.172042627240044e-06, "loss": 0.0895, "num_tokens": 1181661922.0, "reward": 0.616629496216774, "reward_std": 0.15080824121832848, "rewards/accuracy_reward/mean": 0.12499999720603228, "rewards/accuracy_reward/std": 0.3089786507189274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0387524738907814, "step": 2450 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 876.3817443847656, "completions/mean_terminated_length": 757.9182586669922, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "epoch": 0.7321335225151221, "grad_norm": 0.9543474912643433, "kl": 1.306640625, "learning_rate": 3.1640769797712865e-06, "loss": 0.0727, "num_tokens": 1182129533.0, "reward": 0.5496651977300644, "reward_std": 0.0688771833665669, "rewards/accuracy_reward/mean": 0.055803573690354824, "rewards/accuracy_reward/std": 0.1863393485546112, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 2451 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 845.3594055175781, "completions/mean_terminated_length": 753.6882781982422, "completions/min_length": 290.25, "completions/min_terminated_length": 290.25, "epoch": 0.7324322306026436, "grad_norm": 0.5099231600761414, "kl": 1.5859375, "learning_rate": 3.1561194664471638e-06, "loss": 0.1047, "num_tokens": 1182576142.0, "reward": 0.7555803954601288, "reward_std": 0.18412843346595764, "rewards/accuracy_reward/mean": 0.2633928544819355, "rewards/accuracy_reward/std": 0.41303521394729614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 2452 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 861.7924499511719, "completions/mean_terminated_length": 757.4574279785156, "completions/min_length": 284.25, "completions/min_terminated_length": 284.25, "epoch": 0.732730938690165, "grad_norm": 0.22780898213386536, "kl": 1.4638671875, "learning_rate": 3.148170096736408e-06, "loss": 0.0846, "num_tokens": 1183041729.0, "reward": 0.7109375298023224, "reward_std": 0.12640821747481823, "rewards/accuracy_reward/mean": 0.21875000558793545, "rewards/accuracy_reward/std": 0.36530330777168274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0410674219019711, "step": 2453 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 863.3906555175781, "completions/mean_terminated_length": 772.8523864746094, "completions/min_length": 280.75, "completions/min_terminated_length": 280.75, "epoch": 0.7330296467776866, "grad_norm": 0.4412737488746643, "kl": 1.4501953125, "learning_rate": 3.140228880098074e-06, "loss": 0.0826, "num_tokens": 1183510752.0, "reward": 0.662388414144516, "reward_std": 0.1581863984465599, "rewards/accuracy_reward/mean": 0.1752232126891613, "rewards/accuracy_reward/std": 0.35881057009100914, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03870266629382968, "step": 2454 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 849.6406555175781, "completions/mean_terminated_length": 771.8348999023438, "completions/min_length": 409.25, "completions/min_terminated_length": 409.25, "epoch": 0.733328354865208, "grad_norm": 0.3574855327606201, "kl": 1.765625, "learning_rate": 3.1322958259815016e-06, "loss": 0.0842, "num_tokens": 1183968079.0, "reward": 0.679129496216774, "reward_std": 0.1948440782725811, "rewards/accuracy_reward/mean": 0.18750000232830644, "rewards/accuracy_reward/std": 0.3496907539665699, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04334343643859029, "step": 2455 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 848.9643249511719, "completions/mean_terminated_length": 747.2667846679688, "completions/min_length": 378.5, "completions/min_terminated_length": 378.5, "epoch": 0.7336270629527294, "grad_norm": 0.2971489727497101, "kl": 1.7373046875, "learning_rate": 3.124370943826326e-06, "loss": 0.0913, "num_tokens": 1184423519.0, "reward": 0.6897321790456772, "reward_std": 0.18209614232182503, "rewards/accuracy_reward/mean": 0.1986607126891613, "rewards/accuracy_reward/std": 0.3991844281554222, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 2456 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2566964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 795.7455749511719, "completions/mean_terminated_length": 716.1042327880859, "completions/min_length": 293.5, "completions/min_terminated_length": 293.5, "epoch": 0.7339257710402509, "grad_norm": 0.6476654410362244, "kl": 2.591796875, "learning_rate": 3.116454243062459e-06, "loss": 0.1635, "num_tokens": 1184845133.0, "reward": 0.713727705180645, "reward_std": 0.1857598703354597, "rewards/accuracy_reward/mean": 0.2254464328289032, "rewards/accuracy_reward/std": 0.3428703173995018, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.05178379639983177, "step": 2457 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27901785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 814.044677734375, "completions/mean_terminated_length": 734.6949462890625, "completions/min_length": 296.25, "completions/min_terminated_length": 296.25, "epoch": 0.7342244791277723, "grad_norm": 0.29991552233695984, "kl": 2.455078125, "learning_rate": 3.1085457331100776e-06, "loss": 0.1277, "num_tokens": 1185279793.0, "reward": 0.7226562798023224, "reward_std": 0.22603745386004448, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.4061707854270935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052092005498707294, "step": 2458 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 837.5000305175781, "completions/mean_terminated_length": 739.0680694580078, "completions/min_length": 264.25, "completions/min_terminated_length": 264.25, "epoch": 0.7345231872152939, "grad_norm": 0.40380677580833435, "kl": 2.12890625, "learning_rate": 3.1006454233796035e-06, "loss": 0.1074, "num_tokens": 1185727217.0, "reward": 0.7511160969734192, "reward_std": 0.136542284861207, "rewards/accuracy_reward/mean": 0.2589285708963871, "rewards/accuracy_reward/std": 0.4295996278524399, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04132169345393777, "step": 2459 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31919642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 814.1830749511719, "completions/mean_terminated_length": 714.8951263427734, "completions/min_length": 369.75, "completions/min_terminated_length": 369.75, "epoch": 0.7348218953028153, "grad_norm": 0.31539085507392883, "kl": 2.35546875, "learning_rate": 3.0927533232717155e-06, "loss": 0.1463, "num_tokens": 1186159731.0, "reward": 0.7109375447034836, "reward_std": 0.16976918652653694, "rewards/accuracy_reward/mean": 0.22098213993012905, "rewards/accuracy_reward/std": 0.38999272882938385, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.050604683347046375, "step": 2460 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.27901785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 801.3928985595703, "completions/mean_terminated_length": 717.3388366699219, "completions/min_length": 276.25, "completions/min_terminated_length": 276.25, "epoch": 0.7351206033903368, "grad_norm": 0.33375370502471924, "kl": 2.6953125, "learning_rate": 3.0848694421773075e-06, "loss": 0.1512, "num_tokens": 1186590867.0, "reward": 0.7561384290456772, "reward_std": 0.21551238000392914, "rewards/accuracy_reward/mean": 0.26785714365541935, "rewards/accuracy_reward/std": 0.4098334163427353, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05163817573338747, "step": 2461 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3214285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 857.3214721679688, "completions/mean_terminated_length": 777.2383880615234, "completions/min_length": 231.0, "completions/min_terminated_length": 231.0, "epoch": 0.7354193114778582, "grad_norm": 0.25655144453048706, "kl": 2.107421875, "learning_rate": 3.0769937894775082e-06, "loss": 0.1126, "num_tokens": 1187042003.0, "reward": 0.6434151977300644, "reward_std": 0.17266026884317398, "rewards/accuracy_reward/mean": 0.1517857159487903, "rewards/accuracy_reward/std": 0.338023342192173, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043343435507267714, "step": 2462 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.0, "completions/mean_length": 808.5134124755859, "completions/mean_terminated_length": 717.4880523681641, "completions/min_length": 389.5, "completions/min_terminated_length": 389.5, "epoch": 0.7357180195653797, "grad_norm": 0.34867265820503235, "kl": 2.455078125, "learning_rate": 3.069126374543643e-06, "loss": 0.1267, "num_tokens": 1187487641.0, "reward": 0.7433035969734192, "reward_std": 0.14665371924638748, "rewards/accuracy_reward/mean": 0.2522321417927742, "rewards/accuracy_reward/std": 0.43058494478464127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589572083204985, "step": 2463 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.5, "completions/mean_length": 874.5469207763672, "completions/mean_terminated_length": 745.5403442382812, "completions/min_length": 365.75, "completions/min_terminated_length": 365.75, "epoch": 0.7360167276529012, "grad_norm": 0.36731767654418945, "kl": 2.91796875, "learning_rate": 3.061267206737244e-06, "loss": 0.1507, "num_tokens": 1187947294.0, "reward": 0.638950914144516, "reward_std": 0.16259825602173805, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.3562110885977745, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 2464 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 845.6451263427734, "completions/mean_terminated_length": 741.4778594970703, "completions/min_length": 374.75, "completions/min_terminated_length": 374.75, "epoch": 0.7363154357404227, "grad_norm": 0.25568175315856934, "kl": 2.654296875, "learning_rate": 3.0534162954100264e-06, "loss": 0.1278, "num_tokens": 1188391391.0, "reward": 0.5474330633878708, "reward_std": 0.08364274469204247, "rewards/accuracy_reward/mean": 0.05803571315482259, "rewards/accuracy_reward/std": 0.17936987802386284, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810389876366, "step": 2465 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 879.8638763427734, "completions/mean_terminated_length": 773.4750366210938, "completions/min_length": 428.75, "completions/min_terminated_length": 428.75, "epoch": 0.7366141438279441, "grad_norm": 0.3547506332397461, "kl": 3.765625, "learning_rate": 3.0455736499038847e-06, "loss": 0.1923, "num_tokens": 1188869602.0, "reward": 0.619419664144516, "reward_std": 0.14347227849066257, "rewards/accuracy_reward/mean": 0.1361607159487903, "rewards/accuracy_reward/std": 0.31216419488191605, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589402794838, "rewards/tag_count_reward/std": 0.06151718832552433, "step": 2466 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 903.3437805175781, "completions/mean_terminated_length": 797.4407501220703, "completions/min_length": 407.75, "completions/min_terminated_length": 407.75, "epoch": 0.7369128519154656, "grad_norm": 0.2062821090221405, "kl": 1.9052734375, "learning_rate": 3.0377392795508687e-06, "loss": 0.0978, "num_tokens": 1189343980.0, "reward": 0.5987723469734192, "reward_std": 0.12231598794460297, "rewards/accuracy_reward/mean": 0.10714285587891936, "rewards/accuracy_reward/std": 0.2850284315645695, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 2467 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 848.0022735595703, "completions/mean_terminated_length": 744.7845764160156, "completions/min_length": 344.75, "completions/min_terminated_length": 344.75, "epoch": 0.737211560002987, "grad_norm": 0.2979012727737427, "kl": 2.625, "learning_rate": 3.0299131936731916e-06, "loss": 0.137, "num_tokens": 1189796669.0, "reward": 0.5831473469734192, "reward_std": 0.12783154100179672, "rewards/accuracy_reward/mean": 0.09598214598372579, "rewards/accuracy_reward/std": 0.2772735096514225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.055795871652662754, "step": 2468 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 831.5357360839844, "completions/mean_terminated_length": 734.0069427490234, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "epoch": 0.7375102680905086, "grad_norm": 0.2310558706521988, "kl": 2.748046875, "learning_rate": 3.0220954015832004e-06, "loss": 0.1581, "num_tokens": 1190237629.0, "reward": 0.7756696790456772, "reward_std": 0.177923196926713, "rewards/accuracy_reward/mean": 0.2857142863795161, "rewards/accuracy_reward/std": 0.4076489955186844, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 2469 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 828.966552734375, "completions/mean_terminated_length": 702.2350158691406, "completions/min_length": 293.5, "completions/min_terminated_length": 293.5, "epoch": 0.73780897617803, "grad_norm": 0.3490762710571289, "kl": 3.44921875, "learning_rate": 3.014285912583378e-06, "loss": 0.1937, "num_tokens": 1190684398.0, "reward": 0.7511161118745804, "reward_std": 0.16277312114834785, "rewards/accuracy_reward/mean": 0.2656249953433871, "rewards/accuracy_reward/std": 0.39672961831092834, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05821291171014309, "step": 2470 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 843.5178833007812, "completions/mean_terminated_length": 742.9985656738281, "completions/min_length": 317.5, "completions/min_terminated_length": 317.5, "epoch": 0.7381076842655515, "grad_norm": 0.46712371706962585, "kl": 2.943359375, "learning_rate": 3.0064847359663284e-06, "loss": 0.1463, "num_tokens": 1191136934.0, "reward": 0.5664062649011612, "reward_std": 0.140758216381073, "rewards/accuracy_reward/mean": 0.07812500023283064, "rewards/accuracy_reward/std": 0.2401261981576681, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05288884975016117, "step": 2471 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 826.4531555175781, "completions/mean_terminated_length": 741.2151489257812, "completions/min_length": 257.5, "completions/min_terminated_length": 257.5, "epoch": 0.7384063923530729, "grad_norm": 0.3703988790512085, "kl": 3.24609375, "learning_rate": 2.998691881014765e-06, "loss": 0.1695, "num_tokens": 1191582033.0, "reward": 0.6411830633878708, "reward_std": 0.2271491102874279, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.3556298241019249, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.0591660775244236, "step": 2472 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 856.591552734375, "completions/mean_terminated_length": 758.2541961669922, "completions/min_length": 233.25, "completions/min_terminated_length": 233.25, "epoch": 0.7387051004405945, "grad_norm": 0.3270806670188904, "kl": 2.4365234375, "learning_rate": 2.990907357001491e-06, "loss": 0.11, "num_tokens": 1192037322.0, "reward": 0.5625000298023224, "reward_std": 0.1510620452463627, "rewards/accuracy_reward/mean": 0.07366071362048388, "rewards/accuracy_reward/std": 0.23011839762330055, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.04892176715657115, "step": 2473 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41517857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 853.7277221679688, "completions/mean_terminated_length": 738.4618682861328, "completions/min_length": 223.25, "completions/min_terminated_length": 223.25, "epoch": 0.7390038085281159, "grad_norm": 0.336352676153183, "kl": 2.251953125, "learning_rate": 2.9831311731894086e-06, "loss": 0.1039, "num_tokens": 1192484720.0, "reward": 0.5758928880095482, "reward_std": 0.11039148364216089, "rewards/accuracy_reward/mean": 0.0848214291036129, "rewards/accuracy_reward/std": 0.23170137032866478, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 2474 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 844.9241485595703, "completions/mean_terminated_length": 747.8793792724609, "completions/min_length": 343.75, "completions/min_terminated_length": 343.75, "epoch": 0.7393025166156374, "grad_norm": 0.3625921607017517, "kl": 2.6640625, "learning_rate": 2.975363338831484e-06, "loss": 0.161, "num_tokens": 1192933070.0, "reward": 0.613839328289032, "reward_std": 0.16874410770833492, "rewards/accuracy_reward/mean": 0.12723214458674192, "rewards/accuracy_reward/std": 0.31473924964666367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05514051392674446, "step": 2475 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 886.5379943847656, "completions/mean_terminated_length": 764.1773834228516, "completions/min_length": 411.0, "completions/min_terminated_length": 411.0, "epoch": 0.7396012247031588, "grad_norm": 0.22995126247406006, "kl": 2.55078125, "learning_rate": 2.967603863170759e-06, "loss": 0.128, "num_tokens": 1193410399.0, "reward": 0.5675223469734192, "reward_std": 0.12526807188987732, "rewards/accuracy_reward/mean": 0.08035714318975806, "rewards/accuracy_reward/std": 0.24614298716187477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 2476 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 831.9107513427734, "completions/mean_terminated_length": 749.6851348876953, "completions/min_length": 292.75, "completions/min_terminated_length": 292.75, "epoch": 0.7398999327906803, "grad_norm": 0.2746545672416687, "kl": 1.9794921875, "learning_rate": 2.9598527554403187e-06, "loss": 0.1244, "num_tokens": 1193858375.0, "reward": 0.7137277126312256, "reward_std": 0.18195547349750996, "rewards/accuracy_reward/mean": 0.22098214784637094, "rewards/accuracy_reward/std": 0.3640187419950962, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 2477 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 846.825927734375, "completions/mean_terminated_length": 734.4071044921875, "completions/min_length": 233.5, "completions/min_terminated_length": 233.5, "epoch": 0.7401986408782018, "grad_norm": 0.6687752604484558, "kl": 2.53125, "learning_rate": 2.9521100248633007e-06, "loss": 0.1513, "num_tokens": 1194313225.0, "reward": 0.6422991454601288, "reward_std": 0.13648154214024544, "rewards/accuracy_reward/mean": 0.15401786006987095, "rewards/accuracy_reward/std": 0.34825318306684494, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050832636654376984, "step": 2478 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 888.0803985595703, "completions/mean_terminated_length": 769.5154418945312, "completions/min_length": 422.0, "completions/min_terminated_length": 422.0, "epoch": 0.7404973489657233, "grad_norm": 0.49018099904060364, "kl": 2.4765625, "learning_rate": 2.944375680652869e-06, "loss": 0.1244, "num_tokens": 1194779725.0, "reward": 0.6841518133878708, "reward_std": 0.16818532533943653, "rewards/accuracy_reward/mean": 0.19419643096625805, "rewards/accuracy_reward/std": 0.3778747469186783, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 2479 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 865.6786041259766, "completions/mean_terminated_length": 765.7321472167969, "completions/min_length": 394.25, "completions/min_terminated_length": 394.25, "epoch": 0.7407960570532447, "grad_norm": 0.2891540229320526, "kl": 2.787109375, "learning_rate": 2.9366497320122133e-06, "loss": 0.1469, "num_tokens": 1195244237.0, "reward": 0.667410746216774, "reward_std": 0.20250565186142921, "rewards/accuracy_reward/mean": 0.1808035671710968, "rewards/accuracy_reward/std": 0.37629005312919617, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05534007400274277, "step": 2480 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 857.6027069091797, "completions/mean_terminated_length": 749.4443969726562, "completions/min_length": 303.25, "completions/min_terminated_length": 303.25, "epoch": 0.7410947651407662, "grad_norm": 0.3137862980365753, "kl": 2.755859375, "learning_rate": 2.9289321881345257e-06, "loss": 0.1517, "num_tokens": 1195699371.0, "reward": 0.7756696790456772, "reward_std": 0.19597668573260307, "rewards/accuracy_reward/mean": 0.2901785746216774, "rewards/accuracy_reward/std": 0.44579920917749405, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05846718233078718, "step": 2481 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38169642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 855.5647735595703, "completions/mean_terminated_length": 754.4489440917969, "completions/min_length": 393.5, "completions/min_terminated_length": 393.5, "epoch": 0.7413934732282876, "grad_norm": 0.25903353095054626, "kl": 2.49609375, "learning_rate": 2.9212230582030034e-06, "loss": 0.138, "num_tokens": 1196146472.0, "reward": 0.6456473395228386, "reward_std": 0.1456722691655159, "rewards/accuracy_reward/mean": 0.15625, "rewards/accuracy_reward/std": 0.2997797802090645, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.489397332072258, "rewards/tag_count_reward/std": 0.050546927377581596, "step": 2482 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37276785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 858.6161041259766, "completions/mean_terminated_length": 762.2463531494141, "completions/min_length": 240.75, "completions/min_terminated_length": 240.75, "epoch": 0.7416921813158092, "grad_norm": 0.268311470746994, "kl": 2.80859375, "learning_rate": 2.913522351390834e-06, "loss": 0.1483, "num_tokens": 1196606076.0, "reward": 0.6456473469734192, "reward_std": 0.15503436140716076, "rewards/accuracy_reward/mean": 0.15848214039579034, "rewards/accuracy_reward/std": 0.3320845030248165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05500977113842964, "step": 2483 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 848.3170166015625, "completions/mean_terminated_length": 736.7417144775391, "completions/min_length": 316.5, "completions/min_terminated_length": 316.5, "epoch": 0.7419908894033306, "grad_norm": 0.4465738832950592, "kl": 2.0546875, "learning_rate": 2.9058300768611704e-06, "loss": 0.1242, "num_tokens": 1197055898.0, "reward": 0.6333705633878708, "reward_std": 0.18762502446770668, "rewards/accuracy_reward/mean": 0.1428571455180645, "rewards/accuracy_reward/std": 0.28639284521341324, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241139769554, "step": 2484 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428564, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 893.8147735595703, "completions/mean_terminated_length": 766.3231048583984, "completions/min_length": 351.75, "completions/min_terminated_length": 351.75, "epoch": 0.7422895974908521, "grad_norm": 0.17957362532615662, "kl": 2.2041015625, "learning_rate": 2.898146243767146e-06, "loss": 0.1129, "num_tokens": 1197532583.0, "reward": 0.6495535969734192, "reward_std": 0.16774838231503963, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.3330972418189049, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392761349678, "rewards/tag_count_reward/std": 0.050723335705697536, "step": 2485 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 874.0848693847656, "completions/mean_terminated_length": 759.0364379882812, "completions/min_length": 389.75, "completions/min_terminated_length": 389.75, "epoch": 0.7425883055783735, "grad_norm": 0.2634789049625397, "kl": 2.1240234375, "learning_rate": 2.8904708612518404e-06, "loss": 0.113, "num_tokens": 1197997501.0, "reward": 0.6099330633878708, "reward_std": 0.13510527834296227, "rewards/accuracy_reward/mean": 0.11830356949940324, "rewards/accuracy_reward/std": 0.2964170165359974, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 2486 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 876.6027221679688, "completions/mean_terminated_length": 756.4450378417969, "completions/min_length": 209.5, "completions/min_terminated_length": 209.5, "epoch": 0.742887013665895, "grad_norm": 0.4137159585952759, "kl": 3.169921875, "learning_rate": 2.8828039384482874e-06, "loss": 0.1536, "num_tokens": 1198469675.0, "reward": 0.536272332072258, "reward_std": 0.08155704964883626, "rewards/accuracy_reward/mean": 0.0513392873108387, "rewards/accuracy_reward/std": 0.14866230264306068, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05607155570760369, "step": 2487 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 902.7991485595703, "completions/mean_terminated_length": 802.9555969238281, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "epoch": 0.7431857217534165, "grad_norm": 0.2810901701450348, "kl": 2.64453125, "learning_rate": 2.875145484479439e-06, "loss": 0.1249, "num_tokens": 1198954881.0, "reward": 0.5619419887661934, "reward_std": 0.12404676340520382, "rewards/accuracy_reward/mean": 0.07589285727590322, "rewards/accuracy_reward/std": 0.16528521478176117, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.0569901280105114, "step": 2488 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 873.3170166015625, "completions/mean_terminated_length": 737.2164001464844, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.743484429840938, "grad_norm": 0.21977481245994568, "kl": 2.6796875, "learning_rate": 2.867495508458186e-06, "loss": 0.1282, "num_tokens": 1199416863.0, "reward": 0.6222098544239998, "reward_std": 0.1741427779197693, "rewards/accuracy_reward/mean": 0.13653273996897042, "rewards/accuracy_reward/std": 0.27824272960424423, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.055920460261404514, "step": 2489 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 884.3437957763672, "completions/mean_terminated_length": 769.3493194580078, "completions/min_length": 314.75, "completions/min_terminated_length": 314.75, "epoch": 0.7437831379284594, "grad_norm": 0.2744370102882385, "kl": 3.22265625, "learning_rate": 2.859854019487318e-06, "loss": 0.1697, "num_tokens": 1199894601.0, "reward": 0.5848214477300644, "reward_std": 0.14813190884888172, "rewards/accuracy_reward/mean": 0.10044642840512097, "rewards/accuracy_reward/std": 0.2740158922970295, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.060609862208366394, "step": 2490 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 850.9107513427734, "completions/mean_terminated_length": 704.4474945068359, "completions/min_length": 297.0, "completions/min_terminated_length": 297.0, "epoch": 0.7440818460159809, "grad_norm": 0.32156768441200256, "kl": 2.40234375, "learning_rate": 2.8522210266595386e-06, "loss": 0.1422, "num_tokens": 1200339793.0, "reward": 0.5602678805589676, "reward_std": 0.13045022264122963, "rewards/accuracy_reward/mean": 0.07403273787349463, "rewards/accuracy_reward/std": 0.25578252598643303, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051120287738740444, "step": 2491 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 862.8728179931641, "completions/mean_terminated_length": 734.0692901611328, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.7443805541035023, "grad_norm": 0.35510653257369995, "kl": 2.384765625, "learning_rate": 2.844596539057428e-06, "loss": 0.1245, "num_tokens": 1200809720.0, "reward": 0.671316996216774, "reward_std": 0.2243034392595291, "rewards/accuracy_reward/mean": 0.1830357164144516, "rewards/accuracy_reward/std": 0.3846605569124222, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.0513117304071784, "step": 2492 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 843.6473693847656, "completions/mean_terminated_length": 729.3484802246094, "completions/min_length": 322.25, "completions/min_terminated_length": 322.25, "epoch": 0.7446792621910239, "grad_norm": 0.4415951073169708, "kl": 2.85546875, "learning_rate": 2.8369805657534576e-06, "loss": 0.1614, "num_tokens": 1201262298.0, "reward": 0.6261160969734192, "reward_std": 0.1743292324244976, "rewards/accuracy_reward/mean": 0.13839285960420966, "rewards/accuracy_reward/std": 0.3156762644648552, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05349547974765301, "step": 2493 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 841.716552734375, "completions/mean_terminated_length": 696.8310241699219, "completions/min_length": 341.5, "completions/min_terminated_length": 341.5, "epoch": 0.7449779702785453, "grad_norm": 0.525810182094574, "kl": 3.052734375, "learning_rate": 2.8293731158099625e-06, "loss": 0.1543, "num_tokens": 1201712987.0, "reward": 0.7220982611179352, "reward_std": 0.19010287150740623, "rewards/accuracy_reward/mean": 0.2477678582072258, "rewards/accuracy_reward/std": 0.42502913624048233, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05168584827333689, "step": 2494 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 869.8817443847656, "completions/mean_terminated_length": 742.4149169921875, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.7452766783660668, "grad_norm": 0.24394752085208893, "kl": 2.33984375, "learning_rate": 2.821774198279138e-06, "loss": 0.1183, "num_tokens": 1202170038.0, "reward": 0.6445312798023224, "reward_std": 0.14936297666281462, "rewards/accuracy_reward/mean": 0.1540178540162742, "rewards/accuracy_reward/std": 0.2629611939191818, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046577731147408485, "step": 2495 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 850.1897735595703, "completions/mean_terminated_length": 735.0120391845703, "completions/min_length": 314.5, "completions/min_terminated_length": 314.5, "epoch": 0.7455753864535882, "grad_norm": 0.6338952779769897, "kl": 2.90234375, "learning_rate": 2.8141838222030195e-06, "loss": 0.1761, "num_tokens": 1202631947.0, "reward": 0.6177455559372902, "reward_std": 0.19664483703672886, "rewards/accuracy_reward/mean": 0.129464291036129, "rewards/accuracy_reward/std": 0.2729303687810898, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05303261987864971, "step": 2496 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 862.4643249511719, "completions/mean_terminated_length": 738.7680206298828, "completions/min_length": 268.5, "completions/min_terminated_length": 268.5, "epoch": 0.7458740945411098, "grad_norm": 0.2599585950374603, "kl": 2.044921875, "learning_rate": 2.8066019966134907e-06, "loss": 0.1105, "num_tokens": 1203087771.0, "reward": 0.643973246216774, "reward_std": 0.1861793827265501, "rewards/accuracy_reward/mean": 0.15178571245633066, "rewards/accuracy_reward/std": 0.2841643411666155, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0410674219019711, "step": 2497 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 886.9420166015625, "completions/mean_terminated_length": 777.2440490722656, "completions/min_length": 435.5, "completions/min_terminated_length": 435.5, "epoch": 0.7461728026286312, "grad_norm": 0.27921000123023987, "kl": 2.0537109375, "learning_rate": 2.7990287305322484e-06, "loss": 0.1138, "num_tokens": 1203559473.0, "reward": 0.638950914144516, "reward_std": 0.16196544095873833, "rewards/accuracy_reward/mean": 0.14732143189758062, "rewards/accuracy_reward/std": 0.3293979689478874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04203914059326053, "step": 2498 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32142857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 802.9174499511719, "completions/mean_terminated_length": 702.4776916503906, "completions/min_length": 217.0, "completions/min_terminated_length": 217.0, "epoch": 0.7464715107161526, "grad_norm": 0.6849974393844604, "kl": 2.9609375, "learning_rate": 2.791464032970812e-06, "loss": 0.1589, "num_tokens": 1203992700.0, "reward": 0.7075893133878708, "reward_std": 0.18981723673641682, "rewards/accuracy_reward/mean": 0.2187500037252903, "rewards/accuracy_reward/std": 0.3898274824023247, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 2499 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 848.6741485595703, "completions/mean_terminated_length": 745.3482971191406, "completions/min_length": 315.5, "completions/min_terminated_length": 315.5, "epoch": 0.7467702188036741, "grad_norm": 0.3334119915962219, "kl": 2.265625, "learning_rate": 2.7839079129305047e-06, "loss": 0.1296, "num_tokens": 1204443210.0, "reward": 0.6406250298023224, "reward_std": 0.19591953232884407, "rewards/accuracy_reward/mean": 0.1495535704307258, "rewards/accuracy_reward/std": 0.32793374359607697, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04696016339585185, "step": 2500 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 872.075927734375, "completions/mean_terminated_length": 765.8222198486328, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "epoch": 0.7470689268911955, "grad_norm": 0.24093830585479736, "kl": 2.4609375, "learning_rate": 2.776360379402445e-06, "loss": 0.1262, "num_tokens": 1204907276.0, "reward": 0.6328125447034836, "reward_std": 0.1312134899199009, "rewards/accuracy_reward/mean": 0.14285714039579034, "rewards/accuracy_reward/std": 0.30915748700499535, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046435759868472815, "step": 2501 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 834.2232513427734, "completions/mean_terminated_length": 696.7701110839844, "completions/min_length": 344.5, "completions/min_terminated_length": 344.5, "epoch": 0.747367634978717, "grad_norm": 0.26952874660491943, "kl": 2.619140625, "learning_rate": 2.7688214413675253e-06, "loss": 0.1533, "num_tokens": 1205351232.0, "reward": 0.6746652126312256, "reward_std": 0.18949658051133156, "rewards/accuracy_reward/mean": 0.1852678582072258, "rewards/accuracy_reward/std": 0.3758119232952595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 2502 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 838.4799652099609, "completions/mean_terminated_length": 705.7631530761719, "completions/min_length": 263.5, "completions/min_terminated_length": 263.5, "epoch": 0.7476663430662385, "grad_norm": 0.30627062916755676, "kl": 2.30859375, "learning_rate": 2.761291107796421e-06, "loss": 0.1163, "num_tokens": 1205796055.0, "reward": 0.635044664144516, "reward_std": 0.13450738787651062, "rewards/accuracy_reward/mean": 0.14285714272409678, "rewards/accuracy_reward/std": 0.3309754282236099, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 2503 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43080357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 863.5111999511719, "completions/mean_terminated_length": 748.6766052246094, "completions/min_length": 345.75, "completions/min_terminated_length": 345.75, "epoch": 0.74796505115376, "grad_norm": 0.20288094878196716, "kl": 2.287109375, "learning_rate": 2.7537693876495585e-06, "loss": 0.1336, "num_tokens": 1206258444.0, "reward": 0.686941996216774, "reward_std": 0.190759789198637, "rewards/accuracy_reward/mean": 0.1964285708963871, "rewards/accuracy_reward/std": 0.33943646401166916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.044759462121874094, "step": 2504 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 891.3794860839844, "completions/mean_terminated_length": 766.3296966552734, "completions/min_length": 363.25, "completions/min_terminated_length": 363.25, "epoch": 0.7482637592412814, "grad_norm": 0.661066472530365, "kl": 2.671875, "learning_rate": 2.746256289877126e-06, "loss": 0.1263, "num_tokens": 1206733046.0, "reward": 0.5959821790456772, "reward_std": 0.14025280810892582, "rewards/accuracy_reward/mean": 0.10714285587891936, "rewards/accuracy_reward/std": 0.2820703722536564, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2505 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 862.5000457763672, "completions/mean_terminated_length": 754.7121124267578, "completions/min_length": 260.25, "completions/min_terminated_length": 260.25, "epoch": 0.7485624673288029, "grad_norm": 0.3230507969856262, "kl": 2.0751953125, "learning_rate": 2.7387518234190414e-06, "loss": 0.0949, "num_tokens": 1207184454.0, "reward": 0.6506696790456772, "reward_std": 0.15268856473267078, "rewards/accuracy_reward/mean": 0.15848214412108064, "rewards/accuracy_reward/std": 0.32359161600470543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2506 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35267857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 851.3281555175781, "completions/mean_terminated_length": 758.4396514892578, "completions/min_length": 300.5, "completions/min_terminated_length": 300.5, "epoch": 0.7488611754163244, "grad_norm": 0.19714298844337463, "kl": 2.0859375, "learning_rate": 2.7312559972049603e-06, "loss": 0.0991, "num_tokens": 1207637033.0, "reward": 0.5931919813156128, "reward_std": 0.1033236738294363, "rewards/accuracy_reward/mean": 0.10267857415601611, "rewards/accuracy_reward/std": 0.235834501683712, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2507 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 829.2969055175781, "completions/mean_terminated_length": 724.7390441894531, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "epoch": 0.7491598835038459, "grad_norm": 0.26417917013168335, "kl": 2.1171875, "learning_rate": 2.723768820154251e-06, "loss": 0.1288, "num_tokens": 1208078750.0, "reward": 0.7254464626312256, "reward_std": 0.1550130993127823, "rewards/accuracy_reward/mean": 0.2496279771439731, "rewards/accuracy_reward/std": 0.3872815668582916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 2508 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.25, "completions/mean_length": 842.6986846923828, "completions/mean_terminated_length": 733.1296997070312, "completions/min_length": 324.5, "completions/min_terminated_length": 324.5, "epoch": 0.7494585915913673, "grad_norm": 0.5323864221572876, "kl": 1.89453125, "learning_rate": 2.716290301175999e-06, "loss": 0.1107, "num_tokens": 1208538263.0, "reward": 0.643973246216774, "reward_std": 0.12149653024971485, "rewards/accuracy_reward/mean": 0.1517857164144516, "rewards/accuracy_reward/std": 0.34750843420624733, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04272336792200804, "step": 2509 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 887.2545166015625, "completions/mean_terminated_length": 778.0156097412109, "completions/min_length": 308.5, "completions/min_terminated_length": 308.5, "epoch": 0.7497572996788888, "grad_norm": 0.1837080717086792, "kl": 1.673828125, "learning_rate": 2.708820449168974e-06, "loss": 0.0902, "num_tokens": 1209015753.0, "reward": 0.6043526977300644, "reward_std": 0.15732554905116558, "rewards/accuracy_reward/mean": 0.11160714412108064, "rewards/accuracy_reward/std": 0.2964125759899616, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196681171656, "step": 2510 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2946428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 801.2745819091797, "completions/mean_terminated_length": 709.8257598876953, "completions/min_length": 190.75, "completions/min_terminated_length": 190.75, "epoch": 0.7500560077664102, "grad_norm": 0.5712687969207764, "kl": 1.931640625, "learning_rate": 2.7013592730216464e-06, "loss": 0.1313, "num_tokens": 1209444996.0, "reward": 0.7421875298023224, "reward_std": 0.1298266239464283, "rewards/accuracy_reward/mean": 0.2499999962747097, "rewards/accuracy_reward/std": 0.4247829094529152, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 2511 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 808.1986999511719, "completions/mean_terminated_length": 724.4212646484375, "completions/min_length": 294.25, "completions/min_terminated_length": 294.25, "epoch": 0.7503547158539318, "grad_norm": 0.4229508638381958, "kl": 1.7861328125, "learning_rate": 2.69390678161215e-06, "loss": 0.1113, "num_tokens": 1209876653.0, "reward": 0.777901828289032, "reward_std": 0.1757859606295824, "rewards/accuracy_reward/mean": 0.2857142873108387, "rewards/accuracy_reward/std": 0.44496677815914154, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 2512 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2901785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 806.9844207763672, "completions/mean_terminated_length": 716.9052124023438, "completions/min_length": 251.5, "completions/min_terminated_length": 251.5, "epoch": 0.7506534239414532, "grad_norm": 0.2820585072040558, "kl": 2.564453125, "learning_rate": 2.6864629838082957e-06, "loss": 0.1642, "num_tokens": 1210308342.0, "reward": 0.6902901977300644, "reward_std": 0.21845932863652706, "rewards/accuracy_reward/mean": 0.20386904664337635, "rewards/accuracy_reward/std": 0.38831964135169983, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.050059826113283634, "step": 2513 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 864.3995819091797, "completions/mean_terminated_length": 721.9417724609375, "completions/min_length": 329.25, "completions/min_terminated_length": 329.25, "epoch": 0.7509521320289747, "grad_norm": 0.37925028800964355, "kl": 2.1513671875, "learning_rate": 2.679027888467545e-06, "loss": 0.0974, "num_tokens": 1210763193.0, "reward": 0.6350446790456772, "reward_std": 0.16570147685706615, "rewards/accuracy_reward/mean": 0.14508928637951612, "rewards/accuracy_reward/std": 0.32685157656669617, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04558529471978545, "step": 2514 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5066964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 870.6652374267578, "completions/mean_terminated_length": 711.1325225830078, "completions/min_length": 335.0, "completions/min_terminated_length": 335.0, "epoch": 0.7512508401164961, "grad_norm": 0.2632642090320587, "kl": 2.421875, "learning_rate": 2.671601504437007e-06, "loss": 0.1224, "num_tokens": 1211222003.0, "reward": 0.7594866454601288, "reward_std": 0.18173546344041824, "rewards/accuracy_reward/mean": 0.2700892873108387, "rewards/accuracy_reward/std": 0.42889759689569473, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04940675385296345, "step": 2515 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 862.1786041259766, "completions/mean_terminated_length": 716.5238800048828, "completions/min_length": 277.0, "completions/min_terminated_length": 277.0, "epoch": 0.7515495482040176, "grad_norm": 0.30322176218032837, "kl": 1.3583984375, "learning_rate": 2.664183840553417e-06, "loss": 0.088, "num_tokens": 1211676227.0, "reward": 0.6651786118745804, "reward_std": 0.19385500252246857, "rewards/accuracy_reward/mean": 0.16964285913854837, "rewards/accuracy_reward/std": 0.345296747982502, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 2516 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.75, "completions/mean_length": 862.6295013427734, "completions/mean_terminated_length": 733.4400177001953, "completions/min_length": 363.5, "completions/min_terminated_length": 363.5, "epoch": 0.7518482562915391, "grad_norm": 0.3354067802429199, "kl": 2.44921875, "learning_rate": 2.656774905643147e-06, "loss": 0.1374, "num_tokens": 1212127149.0, "reward": 0.7047991454601288, "reward_std": 0.18017974123358727, "rewards/accuracy_reward/mean": 0.2165178619325161, "rewards/accuracy_reward/std": 0.3891965448856354, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05277834925800562, "step": 2517 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 869.8571929931641, "completions/mean_terminated_length": 759.0793304443359, "completions/min_length": 409.75, "completions/min_terminated_length": 409.75, "epoch": 0.7521469643790606, "grad_norm": 0.42613670229911804, "kl": 2.59375, "learning_rate": 2.6493747085221676e-06, "loss": 0.1443, "num_tokens": 1212595213.0, "reward": 0.6824777126312256, "reward_std": 0.20031826198101044, "rewards/accuracy_reward/mean": 0.1941964328289032, "rewards/accuracy_reward/std": 0.3907657489180565, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 2518 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 897.1362152099609, "completions/mean_terminated_length": 774.9215393066406, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "epoch": 0.752445672466582, "grad_norm": 0.24372966587543488, "kl": 1.419921875, "learning_rate": 2.641983257996067e-06, "loss": 0.0701, "num_tokens": 1213066298.0, "reward": 0.6724330484867096, "reward_std": 0.17501682043075562, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.37037649750709534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.030261989682912827, "step": 2519 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49330357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 897.9286193847656, "completions/mean_terminated_length": 784.3044128417969, "completions/min_length": 382.25, "completions/min_terminated_length": 382.25, "epoch": 0.7527443805541035, "grad_norm": 0.2525337338447571, "kl": 2.205078125, "learning_rate": 2.634600562860009e-06, "loss": 0.1216, "num_tokens": 1213538474.0, "reward": 0.5987723469734192, "reward_std": 0.12692834297195077, "rewards/accuracy_reward/mean": 0.10937500232830644, "rewards/accuracy_reward/std": 0.23684940859675407, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2520 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 840.3973541259766, "completions/mean_terminated_length": 712.0863952636719, "completions/min_length": 300.25, "completions/min_terminated_length": 300.25, "epoch": 0.753043088641625, "grad_norm": 0.4180089235305786, "kl": 2.0126953125, "learning_rate": 2.6272266318987606e-06, "loss": 0.109, "num_tokens": 1213988540.0, "reward": 0.6367187947034836, "reward_std": 0.12654871866106987, "rewards/accuracy_reward/mean": 0.14806547900661826, "rewards/accuracy_reward/std": 0.3297814838588238, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.0494418740272522, "step": 2521 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5714285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 910.6942443847656, "completions/mean_terminated_length": 766.1851959228516, "completions/min_length": 387.25, "completions/min_terminated_length": 387.25, "epoch": 0.7533417967291465, "grad_norm": 0.2002362310886383, "kl": 1.6484375, "learning_rate": 2.6198614738866402e-06, "loss": 0.0876, "num_tokens": 1214471635.0, "reward": 0.7388393133878708, "reward_std": 0.21602420508861542, "rewards/accuracy_reward/mean": 0.2455357164144516, "rewards/accuracy_reward/std": 0.4204942062497139, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03475248906761408, "step": 2522 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 827.4486999511719, "completions/mean_terminated_length": 698.5095062255859, "completions/min_length": 299.75, "completions/min_terminated_length": 299.75, "epoch": 0.7536405048166679, "grad_norm": 0.28049585223197937, "kl": 2.2119140625, "learning_rate": 2.61250509758754e-06, "loss": 0.1301, "num_tokens": 1214916764.0, "reward": 0.7449777126312256, "reward_std": 0.22422588989138603, "rewards/accuracy_reward/mean": 0.2544642835855484, "rewards/accuracy_reward/std": 0.42752181738615036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04537529917433858, "step": 2523 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 905.6920166015625, "completions/mean_terminated_length": 770.5204010009766, "completions/min_length": 420.0, "completions/min_terminated_length": 420.0, "epoch": 0.7539392129041894, "grad_norm": 0.4101385474205017, "kl": 2.1435546875, "learning_rate": 2.605157511754892e-06, "loss": 0.0967, "num_tokens": 1215399714.0, "reward": 0.577566996216774, "reward_std": 0.0788487121462822, "rewards/accuracy_reward/mean": 0.08705356996506453, "rewards/accuracy_reward/std": 0.24672169983386993, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.044759462121874094, "step": 2524 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 811.4107360839844, "completions/mean_terminated_length": 724.4310607910156, "completions/min_length": 240.0, "completions/min_terminated_length": 240.0, "epoch": 0.7542379209917108, "grad_norm": 0.34440356492996216, "kl": 2.01171875, "learning_rate": 2.5978187251316823e-06, "loss": 0.1119, "num_tokens": 1215847018.0, "reward": 0.7561384290456772, "reward_std": 0.17700709775090218, "rewards/accuracy_reward/mean": 0.2656250037252903, "rewards/accuracy_reward/std": 0.42662961781024933, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2525 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 869.9464721679688, "completions/mean_terminated_length": 753.2842254638672, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "epoch": 0.7545366290792324, "grad_norm": 0.20472320914268494, "kl": 2.201171875, "learning_rate": 2.5904887464504115e-06, "loss": 0.1099, "num_tokens": 1216302210.0, "reward": 0.6171875149011612, "reward_std": 0.18190312758088112, "rewards/accuracy_reward/mean": 0.12723214086145163, "rewards/accuracy_reward/std": 0.32067856937646866, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 2526 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34821428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 797.1920013427734, "completions/mean_terminated_length": 676.0203247070312, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "epoch": 0.7548353371667538, "grad_norm": 0.4843668043613434, "kl": 3.26953125, "learning_rate": 2.5831675844331094e-06, "loss": 0.1674, "num_tokens": 1216733880.0, "reward": 0.6311384290456772, "reward_std": 0.1297632958739996, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.348141685128212, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05715245008468628, "step": 2527 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 838.7277221679688, "completions/mean_terminated_length": 715.1860198974609, "completions/min_length": 263.5, "completions/min_terminated_length": 263.5, "epoch": 0.7551340452542753, "grad_norm": 0.34584832191467285, "kl": 2.61328125, "learning_rate": 2.5758552477913123e-06, "loss": 0.1422, "num_tokens": 1217178878.0, "reward": 0.659598246216774, "reward_std": 0.15907744504511356, "rewards/accuracy_reward/mean": 0.16964285541325808, "rewards/accuracy_reward/std": 0.35308296233415604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 2528 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 894.8861999511719, "completions/mean_terminated_length": 774.1568908691406, "completions/min_length": 396.5, "completions/min_terminated_length": 396.5, "epoch": 0.7554327533417967, "grad_norm": 0.3179720938205719, "kl": 1.412109375, "learning_rate": 2.5685517452260566e-06, "loss": 0.0819, "num_tokens": 1217648539.0, "reward": 0.6690848469734192, "reward_std": 0.14393910020589828, "rewards/accuracy_reward/mean": 0.17410713993012905, "rewards/accuracy_reward/std": 0.3710886090993881, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 2529 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 878.5603179931641, "completions/mean_terminated_length": 748.314697265625, "completions/min_length": 399.0, "completions/min_terminated_length": 399.0, "epoch": 0.7557314614293182, "grad_norm": 0.20256011188030243, "kl": 1.7822265625, "learning_rate": 2.5612570854278664e-06, "loss": 0.1036, "num_tokens": 1218118198.0, "reward": 0.6266741305589676, "reward_std": 0.15864091366529465, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.3358486145734787, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.038913180120289326, "step": 2530 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 886.5803985595703, "completions/mean_terminated_length": 761.5140380859375, "completions/min_length": 408.5, "completions/min_terminated_length": 408.5, "epoch": 0.7560301695168397, "grad_norm": 0.403116375207901, "kl": 1.90625, "learning_rate": 2.5539712770767377e-06, "loss": 0.097, "num_tokens": 1218582122.0, "reward": 0.6886161118745804, "reward_std": 0.13911988958716393, "rewards/accuracy_reward/mean": 0.19642857741564512, "rewards/accuracy_reward/std": 0.37566712498664856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.041961644776165485, "step": 2531 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 835.1585083007812, "completions/mean_terminated_length": 729.8908538818359, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "epoch": 0.7563288776043612, "grad_norm": 0.385601282119751, "kl": 2.173828125, "learning_rate": 2.546694328842144e-06, "loss": 0.1243, "num_tokens": 1219022049.0, "reward": 0.7276785969734192, "reward_std": 0.1933701653033495, "rewards/accuracy_reward/mean": 0.23660714365541935, "rewards/accuracy_reward/std": 0.4043549373745918, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 2532 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4866071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 887.0424499511719, "completions/mean_terminated_length": 770.2510681152344, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.7566275856918826, "grad_norm": 0.3709987998008728, "kl": 2.7578125, "learning_rate": 2.539426249383006e-06, "loss": 0.1457, "num_tokens": 1219485476.0, "reward": 0.6406250298023224, "reward_std": 0.15686378814280033, "rewards/accuracy_reward/mean": 0.15252975886687636, "rewards/accuracy_reward/std": 0.33201291784644127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 2533 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39508928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 855.5893249511719, "completions/mean_terminated_length": 755.0949859619141, "completions/min_length": 309.25, "completions/min_terminated_length": 309.25, "epoch": 0.7569262937794041, "grad_norm": 0.29100173711776733, "kl": 1.208984375, "learning_rate": 2.532167047347698e-06, "loss": 0.0647, "num_tokens": 1219939564.0, "reward": 0.738839328289032, "reward_std": 0.23076428845524788, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.42036695778369904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 2534 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42187499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 827.310302734375, "completions/mean_terminated_length": 696.1716766357422, "completions/min_length": 190.75, "completions/min_terminated_length": 190.75, "epoch": 0.7572250018669255, "grad_norm": 0.25293800234794617, "kl": 3.080078125, "learning_rate": 2.5249167313740307e-06, "loss": 0.1793, "num_tokens": 1220382391.0, "reward": 0.6774553805589676, "reward_std": 0.1543068839237094, "rewards/accuracy_reward/mean": 0.18973214668221772, "rewards/accuracy_reward/std": 0.3485171105712652, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 2535 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 862.1763763427734, "completions/mean_terminated_length": 749.7566986083984, "completions/min_length": 332.0, "completions/min_terminated_length": 332.0, "epoch": 0.7575237099544471, "grad_norm": 0.33962875604629517, "kl": 2.0146484375, "learning_rate": 2.5176753100892426e-06, "loss": 0.0931, "num_tokens": 1220833606.0, "reward": 0.6344866454601288, "reward_std": 0.15697863139212132, "rewards/accuracy_reward/mean": 0.14285714458674192, "rewards/accuracy_reward/std": 0.33028651773929596, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04203914059326053, "step": 2536 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 865.2143249511719, "completions/mean_terminated_length": 745.1133728027344, "completions/min_length": 291.5, "completions/min_terminated_length": 291.5, "epoch": 0.7578224180419685, "grad_norm": 0.5977761149406433, "kl": 2.666015625, "learning_rate": 2.5104427921099783e-06, "loss": 0.1049, "num_tokens": 1221298454.0, "reward": 0.5719866305589676, "reward_std": 0.1501995548605919, "rewards/accuracy_reward/mean": 0.08482142863795161, "rewards/accuracy_reward/std": 0.2462320774793625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05430487543344498, "step": 2537 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 854.2545013427734, "completions/mean_terminated_length": 726.4608764648438, "completions/min_length": 332.5, "completions/min_terminated_length": 332.5, "epoch": 0.75812112612949, "grad_norm": 0.22624574601650238, "kl": 1.7265625, "learning_rate": 2.5032191860423016e-06, "loss": 0.1078, "num_tokens": 1221753160.0, "reward": 0.7477678805589676, "reward_std": 0.21953085064888, "rewards/accuracy_reward/mean": 0.25446428544819355, "rewards/accuracy_reward/std": 0.41701045632362366, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 2538 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 898.5893249511719, "completions/mean_terminated_length": 781.1317901611328, "completions/min_length": 349.25, "completions/min_terminated_length": 349.25, "epoch": 0.7584198342170114, "grad_norm": 0.22433628141880035, "kl": 2.134765625, "learning_rate": 2.496004500481661e-06, "loss": 0.1092, "num_tokens": 1222221152.0, "reward": 0.6579241454601288, "reward_std": 0.15296828374266624, "rewards/accuracy_reward/mean": 0.16741071362048388, "rewards/accuracy_reward/std": 0.3434828296303749, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983776688576, "step": 2539 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 874.0178985595703, "completions/mean_terminated_length": 749.3433685302734, "completions/min_length": 318.25, "completions/min_terminated_length": 318.25, "epoch": 0.758718542304533, "grad_norm": 0.2697027921676636, "kl": 1.873046875, "learning_rate": 2.4887987440129e-06, "loss": 0.1025, "num_tokens": 1222687192.0, "reward": 0.7265625447034836, "reward_std": 0.19943225756287575, "rewards/accuracy_reward/mean": 0.2366071417927742, "rewards/accuracy_reward/std": 0.33358221501111984, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04863459337502718, "step": 2540 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.53125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 882.7009429931641, "completions/mean_terminated_length": 725.8917388916016, "completions/min_length": 349.75, "completions/min_terminated_length": 349.75, "epoch": 0.7590172503920544, "grad_norm": 0.2909044921398163, "kl": 2.4951171875, "learning_rate": 2.4816019252102274e-06, "loss": 0.1133, "num_tokens": 1223168146.0, "reward": 0.6456473469734192, "reward_std": 0.1403108574450016, "rewards/accuracy_reward/mean": 0.15848214272409678, "rewards/accuracy_reward/std": 0.3288809396326542, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.047371624037623405, "step": 2541 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 828.9687805175781, "completions/mean_terminated_length": 698.3993225097656, "completions/min_length": 312.75, "completions/min_terminated_length": 312.75, "epoch": 0.7593159584795758, "grad_norm": 0.29662179946899414, "kl": 2.0224609375, "learning_rate": 2.474414052637224e-06, "loss": 0.1114, "num_tokens": 1223606372.0, "reward": 0.689732164144516, "reward_std": 0.1746416538953781, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.3869904913008213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04951790627092123, "step": 2542 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 901.5223846435547, "completions/mean_terminated_length": 776.3787384033203, "completions/min_length": 374.25, "completions/min_terminated_length": 374.25, "epoch": 0.7596146665670973, "grad_norm": 0.17747190594673157, "kl": 1.80859375, "learning_rate": 2.4672351348468225e-06, "loss": 0.0751, "num_tokens": 1224076110.0, "reward": 0.6255580484867096, "reward_std": 0.13992240279912949, "rewards/accuracy_reward/mean": 0.1339285708963871, "rewards/accuracy_reward/std": 0.32919422164559364, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2543 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43303571428571436, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 860.997802734375, "completions/mean_terminated_length": 743.5950317382812, "completions/min_length": 342.25, "completions/min_terminated_length": 342.25, "epoch": 0.7599133746546187, "grad_norm": 0.34557050466537476, "kl": 2.203125, "learning_rate": 2.4600651803813057e-06, "loss": 0.1189, "num_tokens": 1224533325.0, "reward": 0.5993303805589676, "reward_std": 0.15933484211564064, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.304813664406538, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04863459337502718, "step": 2544 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 856.8348541259766, "completions/mean_terminated_length": 707.9260864257812, "completions/min_length": 263.5, "completions/min_terminated_length": 263.5, "epoch": 0.7602120827421402, "grad_norm": 0.2247348576784134, "kl": 1.3876953125, "learning_rate": 2.45290419777228e-06, "loss": 0.0909, "num_tokens": 1224997315.0, "reward": 0.7840401977300644, "reward_std": 0.21089236810803413, "rewards/accuracy_reward/mean": 0.2901785671710968, "rewards/accuracy_reward/std": 0.435589537024498, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03343775775283575, "step": 2545 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48883928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 881.5893249511719, "completions/mean_terminated_length": 763.8821716308594, "completions/min_length": 347.0, "completions/min_terminated_length": 347.0, "epoch": 0.7605107908296617, "grad_norm": 0.29071444272994995, "kl": 2.59375, "learning_rate": 2.4457521955406872e-06, "loss": 0.1247, "num_tokens": 1225459051.0, "reward": 0.5602678954601288, "reward_std": 0.14442481007426977, "rewards/accuracy_reward/mean": 0.0736607147846371, "rewards/accuracy_reward/std": 0.23506737314164639, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05434367246925831, "step": 2546 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 876.0826416015625, "completions/mean_terminated_length": 766.7057037353516, "completions/min_length": 404.5, "completions/min_terminated_length": 404.5, "epoch": 0.7608094989171832, "grad_norm": 0.21281790733337402, "kl": 1.3837890625, "learning_rate": 2.438609182196773e-06, "loss": 0.0754, "num_tokens": 1225927840.0, "reward": 0.6344866305589676, "reward_std": 0.13449226319789886, "rewards/accuracy_reward/mean": 0.14508928172290325, "rewards/accuracy_reward/std": 0.33669375255703926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 2547 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 895.169677734375, "completions/mean_terminated_length": 795.0995330810547, "completions/min_length": 334.25, "completions/min_terminated_length": 334.25, "epoch": 0.7611082070047046, "grad_norm": 0.3630477786064148, "kl": 2.166015625, "learning_rate": 2.431475166240096e-06, "loss": 0.1157, "num_tokens": 1226398476.0, "reward": 0.5864955633878708, "reward_std": 0.13533868081867695, "rewards/accuracy_reward/mean": 0.09821428498253226, "rewards/accuracy_reward/std": 0.2840775027871132, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052092005498707294, "step": 2548 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 826.7500305175781, "completions/mean_terminated_length": 726.2324829101562, "completions/min_length": 373.5, "completions/min_terminated_length": 373.5, "epoch": 0.7614069150922261, "grad_norm": 0.3754345774650574, "kl": 2.32421875, "learning_rate": 2.4243501561595027e-06, "loss": 0.1355, "num_tokens": 1226834172.0, "reward": 0.6378348469734192, "reward_std": 0.16462285444140434, "rewards/accuracy_reward/mean": 0.14955357369035482, "rewards/accuracy_reward/std": 0.32198499888181686, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05223577655851841, "step": 2549 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2991071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 835.5558471679688, "completions/mean_terminated_length": 753.9723358154297, "completions/min_length": 302.0, "completions/min_terminated_length": 302.0, "epoch": 0.7617056231797475, "grad_norm": 0.2846965491771698, "kl": 1.513671875, "learning_rate": 2.4172341604331317e-06, "loss": 0.0769, "num_tokens": 1227284533.0, "reward": 0.6266741305589676, "reward_std": 0.12616400234401226, "rewards/accuracy_reward/mean": 0.1339285708963871, "rewards/accuracy_reward/std": 0.3398013710975647, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.035923450253903866, "step": 2550 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 871.6384429931641, "completions/mean_terminated_length": 772.9576263427734, "completions/min_length": 275.5, "completions/min_terminated_length": 275.5, "epoch": 0.7620043312672691, "grad_norm": 0.213384211063385, "kl": 2.24609375, "learning_rate": 2.4101271875283818e-06, "loss": 0.1056, "num_tokens": 1227738723.0, "reward": 0.675223246216774, "reward_std": 0.11570547893643379, "rewards/accuracy_reward/mean": 0.1852678619325161, "rewards/accuracy_reward/std": 0.3834473788738251, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764227330685, "step": 2551 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 868.7567291259766, "completions/mean_terminated_length": 755.4157104492188, "completions/min_length": 302.75, "completions/min_terminated_length": 302.75, "epoch": 0.7623030393547905, "grad_norm": 0.24805447459220886, "kl": 2.419921875, "learning_rate": 2.403029245901929e-06, "loss": 0.1244, "num_tokens": 1228213558.0, "reward": 0.6919643133878708, "reward_std": 0.1692141778767109, "rewards/accuracy_reward/mean": 0.20535714365541935, "rewards/accuracy_reward/std": 0.3937135562300682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05632450245320797, "step": 2552 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 858.5848693847656, "completions/mean_terminated_length": 733.4870910644531, "completions/min_length": 407.75, "completions/min_terminated_length": 407.75, "epoch": 0.762601747442312, "grad_norm": 0.19335058331489563, "kl": 1.939453125, "learning_rate": 2.395940343999691e-06, "loss": 0.1022, "num_tokens": 1228675692.0, "reward": 0.6311384290456772, "reward_std": 0.16762850061058998, "rewards/accuracy_reward/mean": 0.14062499813735485, "rewards/accuracy_reward/std": 0.3478466793894768, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681240953505039, "step": 2553 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 858.8571624755859, "completions/mean_terminated_length": 744.700439453125, "completions/min_length": 353.25, "completions/min_terminated_length": 353.25, "epoch": 0.7629004555298334, "grad_norm": 0.3825276494026184, "kl": 2.634765625, "learning_rate": 2.3888604902568426e-06, "loss": 0.1359, "num_tokens": 1229131484.0, "reward": 0.6166294813156128, "reward_std": 0.16225317120552063, "rewards/accuracy_reward/mean": 0.12946428451687098, "rewards/accuracy_reward/std": 0.3186320662498474, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 2554 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 868.1585083007812, "completions/mean_terminated_length": 779.2374572753906, "completions/min_length": 353.75, "completions/min_terminated_length": 353.75, "epoch": 0.763199163617355, "grad_norm": 0.3632780611515045, "kl": 2.427734375, "learning_rate": 2.3817896930977755e-06, "loss": 0.1371, "num_tokens": 1229606003.0, "reward": 0.6902902275323868, "reward_std": 0.12755801528692245, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.40959447622299194, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 2555 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44419642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 868.0781707763672, "completions/mean_terminated_length": 745.6544342041016, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "epoch": 0.7634978717048764, "grad_norm": 0.2201152741909027, "kl": 1.82421875, "learning_rate": 2.3747279609361197e-06, "loss": 0.0953, "num_tokens": 1230064966.0, "reward": 0.6969866454601288, "reward_std": 0.1899699755012989, "rewards/accuracy_reward/mean": 0.20535714365541935, "rewards/accuracy_reward/std": 0.3900962993502617, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 2556 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 863.7545166015625, "completions/mean_terminated_length": 751.6171112060547, "completions/min_length": 393.25, "completions/min_terminated_length": 393.25, "epoch": 0.7637965797923979, "grad_norm": 0.4731193482875824, "kl": 2.982421875, "learning_rate": 2.3676753021747113e-06, "loss": 0.151, "num_tokens": 1230530808.0, "reward": 0.6227678805589676, "reward_std": 0.191216129809618, "rewards/accuracy_reward/mean": 0.13616071455180645, "rewards/accuracy_reward/std": 0.3310621678829193, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05534007586538792, "step": 2557 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 860.4777221679688, "completions/mean_terminated_length": 758.0951538085938, "completions/min_length": 390.0, "completions/min_terminated_length": 390.0, "epoch": 0.7640952878799193, "grad_norm": 0.34696251153945923, "kl": 2.2451171875, "learning_rate": 2.3606317252055945e-06, "loss": 0.1289, "num_tokens": 1230989102.0, "reward": 0.6651785969734192, "reward_std": 0.16739802155643702, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.2953001447021961, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.052474538795650005, "step": 2558 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36160714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 838.7678833007812, "completions/mean_terminated_length": 737.9180755615234, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.7643939959674408, "grad_norm": 0.20775870978832245, "kl": 2.419921875, "learning_rate": 2.353597238409997e-06, "loss": 0.1481, "num_tokens": 1231437110.0, "reward": 0.6662946790456772, "reward_std": 0.20120332390069962, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.3637813702225685, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2559 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 826.9375305175781, "completions/mean_terminated_length": 714.0371246337891, "completions/min_length": 260.5, "completions/min_terminated_length": 260.5, "epoch": 0.7646927040549623, "grad_norm": 0.19265632331371307, "kl": 1.4833984375, "learning_rate": 2.3465718501583446e-06, "loss": 0.0845, "num_tokens": 1231881786.0, "reward": 0.6819196790456772, "reward_std": 0.1262789461761713, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.377642922103405, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 2560 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 859.3482513427734, "completions/mean_terminated_length": 758.5252532958984, "completions/min_length": 261.25, "completions/min_terminated_length": 261.25, "epoch": 0.7649914121424838, "grad_norm": 0.418756902217865, "kl": 2.83984375, "learning_rate": 2.339555568810221e-06, "loss": 0.1419, "num_tokens": 1232334422.0, "reward": 0.6233259290456772, "reward_std": 0.1564357876777649, "rewards/accuracy_reward/mean": 0.13616071362048388, "rewards/accuracy_reward/std": 0.32300353050231934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.054648205637931824, "step": 2561 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 863.6540679931641, "completions/mean_terminated_length": 762.1532897949219, "completions/min_length": 472.25, "completions/min_terminated_length": 472.25, "epoch": 0.7652901202300052, "grad_norm": 0.46235573291778564, "kl": 2.9296875, "learning_rate": 2.332548402714385e-06, "loss": 0.159, "num_tokens": 1232795675.0, "reward": 0.7500000298023224, "reward_std": 0.23534220457077026, "rewards/accuracy_reward/mean": 0.2633928516879678, "rewards/accuracy_reward/std": 0.38444971293210983, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05643500294536352, "step": 2562 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 901.2701416015625, "completions/mean_terminated_length": 779.8550415039062, "completions/min_length": 378.25, "completions/min_terminated_length": 378.25, "epoch": 0.7655888283175267, "grad_norm": 0.22682377696037292, "kl": 2.099609375, "learning_rate": 2.325550360208747e-06, "loss": 0.1018, "num_tokens": 1233276436.0, "reward": 0.5904018133878708, "reward_std": 0.11956008896231651, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.24466858059167862, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04357414972037077, "step": 2563 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 870.3594055175781, "completions/mean_terminated_length": 759.0515594482422, "completions/min_length": 401.75, "completions/min_terminated_length": 401.75, "epoch": 0.7658875364050481, "grad_norm": 0.5219253897666931, "kl": 3.0, "learning_rate": 2.31856144962036e-06, "loss": 0.1517, "num_tokens": 1233737461.0, "reward": 0.7254464477300644, "reward_std": 0.2135119028389454, "rewards/accuracy_reward/mean": 0.2388392873108387, "rewards/accuracy_reward/std": 0.40856895595788956, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.056091989390552044, "step": 2564 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 835.7143249511719, "completions/mean_terminated_length": 690.6214294433594, "completions/min_length": 211.25, "completions/min_terminated_length": 211.25, "epoch": 0.7661862444925697, "grad_norm": 1252556.625, "kl": 5217.28125, "learning_rate": 2.3115816792654057e-06, "loss": 208.1315, "num_tokens": 1234193941.0, "reward": 0.5876116454601288, "reward_std": 0.1449135635048151, "rewards/accuracy_reward/mean": 0.10937500302679837, "rewards/accuracy_reward/std": 0.2731075119227171, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4782366007566452, "rewards/tag_count_reward/std": 0.07149362936615944, "step": 2565 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 856.0402069091797, "completions/mean_terminated_length": 770.0363006591797, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "epoch": 0.7664849525800911, "grad_norm": 0.2084718495607376, "kl": 1.744140625, "learning_rate": 2.3046110574491986e-06, "loss": 0.0975, "num_tokens": 1234659015.0, "reward": 0.643973246216774, "reward_std": 0.12889131344854832, "rewards/accuracy_reward/mean": 0.14992559584788978, "rewards/accuracy_reward/std": 0.3223783038556576, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 2566 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 854.1406555175781, "completions/mean_terminated_length": 757.4205627441406, "completions/min_length": 401.75, "completions/min_terminated_length": 401.75, "epoch": 0.7667836606676126, "grad_norm": 0.22629089653491974, "kl": 1.7880859375, "learning_rate": 2.29764959246616e-06, "loss": 0.1162, "num_tokens": 1235119030.0, "reward": 0.707589328289032, "reward_std": 0.21678689494729042, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.41056668013334274, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044403897132724524, "step": 2567 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 873.404052734375, "completions/mean_terminated_length": 762.9385833740234, "completions/min_length": 331.25, "completions/min_terminated_length": 331.25, "epoch": 0.767082368755134, "grad_norm": 0.22540795803070068, "kl": 1.6923828125, "learning_rate": 2.2906972925998216e-06, "loss": 0.0827, "num_tokens": 1235584475.0, "reward": 0.572544664144516, "reward_std": 0.09715614095330238, "rewards/accuracy_reward/mean": 0.08035714481957257, "rewards/accuracy_reward/std": 0.24641071818768978, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04052485013380647, "step": 2568 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32589285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 819.5022583007812, "completions/mean_terminated_length": 723.3915252685547, "completions/min_length": 349.75, "completions/min_terminated_length": 349.75, "epoch": 0.7673810768426556, "grad_norm": 0.2191864252090454, "kl": 1.80078125, "learning_rate": 2.2837541661228024e-06, "loss": 0.0991, "num_tokens": 1236028780.0, "reward": 0.6729910969734192, "reward_std": 0.15134568884968758, "rewards/accuracy_reward/mean": 0.18080357275903225, "rewards/accuracy_reward/std": 0.37398088723421097, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 2569 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 879.6920013427734, "completions/mean_terminated_length": 774.0412292480469, "completions/min_length": 438.0, "completions/min_terminated_length": 438.0, "epoch": 0.767679784930177, "grad_norm": 0.30289986729621887, "kl": 1.708984375, "learning_rate": 2.2768202212968117e-06, "loss": 0.096, "num_tokens": 1236495762.0, "reward": 0.6188616305589676, "reward_std": 0.15147214010357857, "rewards/accuracy_reward/mean": 0.12723213993012905, "rewards/accuracy_reward/std": 0.3277057446539402, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2570 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 860.7411041259766, "completions/mean_terminated_length": 761.9356994628906, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "epoch": 0.7679784930176985, "grad_norm": 0.3026082515716553, "kl": 2.13671875, "learning_rate": 2.26989546637263e-06, "loss": 0.1268, "num_tokens": 1236955630.0, "reward": 0.6863839626312256, "reward_std": 0.1868258249014616, "rewards/accuracy_reward/mean": 0.19642857275903225, "rewards/accuracy_reward/std": 0.3679957985877991, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 2571 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 854.9018096923828, "completions/mean_terminated_length": 742.0085296630859, "completions/min_length": 377.25, "completions/min_terminated_length": 377.25, "epoch": 0.7682772011052199, "grad_norm": 0.26483896374702454, "kl": 1.5302734375, "learning_rate": 2.262979909590107e-06, "loss": 0.0796, "num_tokens": 1237410674.0, "reward": 0.6372767984867096, "reward_std": 0.1141987107694149, "rewards/accuracy_reward/mean": 0.1450892868451774, "rewards/accuracy_reward/std": 0.3322361633181572, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 2572 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 864.5312957763672, "completions/mean_terminated_length": 749.7226257324219, "completions/min_length": 305.5, "completions/min_terminated_length": 305.5, "epoch": 0.7685759091927414, "grad_norm": 0.18383203446865082, "kl": 1.4462890625, "learning_rate": 2.256073559178145e-06, "loss": 0.0782, "num_tokens": 1237868848.0, "reward": 0.5619419813156128, "reward_std": 0.09203083626925945, "rewards/accuracy_reward/mean": 0.06696428637951612, "rewards/accuracy_reward/std": 0.20743927732110023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 2573 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.5, "completions/mean_length": 854.7143402099609, "completions/mean_terminated_length": 748.8596496582031, "completions/min_length": 358.25, "completions/min_terminated_length": 358.25, "epoch": 0.7688746172802629, "grad_norm": 0.38852187991142273, "kl": 2.072265625, "learning_rate": 2.2491764233546863e-06, "loss": 0.1074, "num_tokens": 1238328192.0, "reward": 0.6032366454601288, "reward_std": 0.14439258351922035, "rewards/accuracy_reward/mean": 0.11383928777649999, "rewards/accuracy_reward/std": 0.29244107380509377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050366250332444906, "step": 2574 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 866.1585083007812, "completions/mean_terminated_length": 750.1650543212891, "completions/min_length": 383.75, "completions/min_terminated_length": 383.75, "epoch": 0.7691733253677844, "grad_norm": 0.33291172981262207, "kl": 2.326171875, "learning_rate": 2.242288510326719e-06, "loss": 0.1208, "num_tokens": 1238797655.0, "reward": 0.689732164144516, "reward_std": 0.1896667741239071, "rewards/accuracy_reward/mean": 0.20089285634458065, "rewards/accuracy_reward/std": 0.3836037218570709, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051717888563871384, "step": 2575 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 876.7656707763672, "completions/mean_terminated_length": 730.1870880126953, "completions/min_length": 352.5, "completions/min_terminated_length": 352.5, "epoch": 0.7694720334553058, "grad_norm": 0.28273794054985046, "kl": 2.8515625, "learning_rate": 2.2354098282902446e-06, "loss": 0.1398, "num_tokens": 1239263086.0, "reward": 0.5691964626312256, "reward_std": 0.1307217087596655, "rewards/accuracy_reward/mean": 0.08258928474970162, "rewards/accuracy_reward/std": 0.2553784158080816, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 2576 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 847.091552734375, "completions/mean_terminated_length": 740.9348754882812, "completions/min_length": 329.75, "completions/min_terminated_length": 329.75, "epoch": 0.7697707415428273, "grad_norm": 0.32763200998306274, "kl": 1.9052734375, "learning_rate": 2.2285403854302912e-06, "loss": 0.0985, "num_tokens": 1239706759.0, "reward": 0.7226562798023224, "reward_std": 0.19201860204339027, "rewards/accuracy_reward/mean": 0.22991071455180645, "rewards/accuracy_reward/std": 0.38161899521946907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 2577 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 842.825927734375, "completions/mean_terminated_length": 709.117919921875, "completions/min_length": 295.0, "completions/min_terminated_length": 295.0, "epoch": 0.7700694496303487, "grad_norm": 0.36559033393859863, "kl": 2.37109375, "learning_rate": 2.2216801899208886e-06, "loss": 0.1483, "num_tokens": 1240154089.0, "reward": 0.592075914144516, "reward_std": 0.13633525185287, "rewards/accuracy_reward/mean": 0.10565476026386023, "rewards/accuracy_reward/std": 0.29583973437547684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04875553119927645, "step": 2578 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4843750000000001, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 896.7723693847656, "completions/mean_terminated_length": 782.5508880615234, "completions/min_length": 416.5, "completions/min_terminated_length": 416.5, "epoch": 0.7703681577178703, "grad_norm": 0.24341893196105957, "kl": 2.87890625, "learning_rate": 2.2148292499250668e-06, "loss": 0.1424, "num_tokens": 1240636883.0, "reward": 0.6088169813156128, "reward_std": 0.15872785821557045, "rewards/accuracy_reward/mean": 0.12276785681024194, "rewards/accuracy_reward/std": 0.2972012609243393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05713389813899994, "step": 2579 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 891.200927734375, "completions/mean_terminated_length": 770.0845489501953, "completions/min_length": 452.25, "completions/min_terminated_length": 452.25, "epoch": 0.7706668658053917, "grad_norm": 0.19467930495738983, "kl": 1.61328125, "learning_rate": 2.207987573594833e-06, "loss": 0.0791, "num_tokens": 1241102029.0, "reward": 0.6607143133878708, "reward_std": 0.13950642570853233, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3646519333124161, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03934345254674554, "step": 2580 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 839.4576416015625, "completions/mean_terminated_length": 719.7200469970703, "completions/min_length": 295.75, "completions/min_terminated_length": 295.75, "epoch": 0.7709655738929132, "grad_norm": 0.23934417963027954, "kl": 2.23046875, "learning_rate": 2.201155169071184e-06, "loss": 0.1207, "num_tokens": 1241544106.0, "reward": 0.6607143133878708, "reward_std": 0.16014116257429123, "rewards/accuracy_reward/mean": 0.17038690438494086, "rewards/accuracy_reward/std": 0.3314271606504917, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 2581 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 902.4018249511719, "completions/mean_terminated_length": 782.56689453125, "completions/min_length": 441.5, "completions/min_terminated_length": 441.5, "epoch": 0.7712642819804346, "grad_norm": 0.1970391422510147, "kl": 1.9609375, "learning_rate": 2.194332044484071e-06, "loss": 0.097, "num_tokens": 1242016206.0, "reward": 0.6824776977300644, "reward_std": 0.17832253500819206, "rewards/accuracy_reward/mean": 0.19196428917348385, "rewards/accuracy_reward/std": 0.3575316444039345, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2582 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 889.1495819091797, "completions/mean_terminated_length": 778.5929107666016, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.7715629900679561, "grad_norm": 0.41665998101234436, "kl": 2.28125, "learning_rate": 2.1875182079524173e-06, "loss": 0.128, "num_tokens": 1242487121.0, "reward": 0.7148437947034836, "reward_std": 0.19188510812819004, "rewards/accuracy_reward/mean": 0.22544642770662904, "rewards/accuracy_reward/std": 0.36188865825533867, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.049606312066316605, "step": 2583 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.5, "completions/mean_length": 859.8794860839844, "completions/mean_terminated_length": 733.4770202636719, "completions/min_length": 354.25, "completions/min_terminated_length": 354.25, "epoch": 0.7718616981554776, "grad_norm": 0.31291648745536804, "kl": 2.740234375, "learning_rate": 2.1807136675840757e-06, "loss": 0.1518, "num_tokens": 1242934619.0, "reward": 0.6696428805589676, "reward_std": 0.13943810388445854, "rewards/accuracy_reward/mean": 0.1808035708963871, "rewards/accuracy_reward/std": 0.3833705186843872, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 2584 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 840.982177734375, "completions/mean_terminated_length": 729.5301971435547, "completions/min_length": 340.25, "completions/min_terminated_length": 340.25, "epoch": 0.772160406242999, "grad_norm": 0.24679504334926605, "kl": 2.044921875, "learning_rate": 2.173918431475861e-06, "loss": 0.1272, "num_tokens": 1243378995.0, "reward": 0.627232164144516, "reward_std": 0.17678524553775787, "rewards/accuracy_reward/mean": 0.1439732164144516, "rewards/accuracy_reward/std": 0.3382776528596878, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491071417927742, "rewards/tag_count_reward/std": 0.044901167973876, "step": 2585 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4754464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.75, "completions/mean_length": 891.0491485595703, "completions/mean_terminated_length": 770.7906494140625, "completions/min_length": 444.25, "completions/min_terminated_length": 444.25, "epoch": 0.7724591143305205, "grad_norm": 0.455127477645874, "kl": 2.044921875, "learning_rate": 2.1671325077134963e-06, "loss": 0.1056, "num_tokens": 1243848537.0, "reward": 0.6395089477300644, "reward_std": 0.17779697850346565, "rewards/accuracy_reward/mean": 0.14732142654247582, "rewards/accuracy_reward/std": 0.31215780042111874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.03758151177316904, "step": 2586 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43750000000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 884.9442291259766, "completions/mean_terminated_length": 777.9606323242188, "completions/min_length": 302.5, "completions/min_terminated_length": 302.5, "epoch": 0.7727578224180419, "grad_norm": 0.20185688138008118, "kl": 2.701171875, "learning_rate": 2.160355904371635e-06, "loss": 0.1305, "num_tokens": 1244308208.0, "reward": 0.551897332072258, "reward_std": 0.13070358894765377, "rewards/accuracy_reward/mean": 0.06473214575089514, "rewards/accuracy_reward/std": 0.2227746807038784, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.053543152287602425, "step": 2587 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 843.591552734375, "completions/mean_terminated_length": 730.7307434082031, "completions/min_length": 372.75, "completions/min_terminated_length": 372.75, "epoch": 0.7730565305055634, "grad_norm": 0.48335161805152893, "kl": 1.8359375, "learning_rate": 2.153588629513832e-06, "loss": 0.1077, "num_tokens": 1244757449.0, "reward": 0.6590401977300644, "reward_std": 0.13222728855907917, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.36108288168907166, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 2588 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 878.4844055175781, "completions/mean_terminated_length": 735.0245971679688, "completions/min_length": 358.75, "completions/min_terminated_length": 358.75, "epoch": 0.7733552385930849, "grad_norm": 0.5870147943496704, "kl": 2.326171875, "learning_rate": 2.146830691192553e-06, "loss": 0.1147, "num_tokens": 1245220370.0, "reward": 0.7868303954601288, "reward_std": 0.20780745521187782, "rewards/accuracy_reward/mean": 0.2968749962747097, "rewards/accuracy_reward/std": 0.42999834567308426, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 2589 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 863.9486999511719, "completions/mean_terminated_length": 757.0549621582031, "completions/min_length": 332.25, "completions/min_terminated_length": 332.25, "epoch": 0.7736539466806064, "grad_norm": 0.22138817608356476, "kl": 2.31640625, "learning_rate": 2.140082097449141e-06, "loss": 0.1167, "num_tokens": 1245673243.0, "reward": 0.6729910969734192, "reward_std": 0.15722259134054184, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.3857460916042328, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 2590 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 900.0915679931641, "completions/mean_terminated_length": 770.2472229003906, "completions/min_length": 283.0, "completions/min_terminated_length": 283.0, "epoch": 0.7739526547681278, "grad_norm": 0.3102407455444336, "kl": 2.828125, "learning_rate": 2.1333428563138304e-06, "loss": 0.1271, "num_tokens": 1246150116.0, "reward": 0.6071428954601288, "reward_std": 0.16203453950583935, "rewards/accuracy_reward/mean": 0.12053571967408061, "rewards/accuracy_reward/std": 0.28118060156702995, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148889839649, "step": 2591 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34598214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 793.1027069091797, "completions/mean_terminated_length": 671.8451232910156, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 0.7742513628556493, "grad_norm": 0.2926759719848633, "kl": 1.603515625, "learning_rate": 2.1266129758057217e-06, "loss": 0.0995, "num_tokens": 1246576770.0, "reward": 0.7767857313156128, "reward_std": 0.1415158868767321, "rewards/accuracy_reward/mean": 0.2834821492433548, "rewards/accuracy_reward/std": 0.4377084746956825, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2592 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 838.8705749511719, "completions/mean_terminated_length": 716.6855163574219, "completions/min_length": 260.75, "completions/min_terminated_length": 260.75, "epoch": 0.7745500709431707, "grad_norm": 0.2028489112854004, "kl": 1.841796875, "learning_rate": 2.119892463932781e-06, "loss": 0.0919, "num_tokens": 1247026552.0, "reward": 0.6171875298023224, "reward_std": 0.1024376368150115, "rewards/accuracy_reward/mean": 0.12723214295692742, "rewards/accuracy_reward/std": 0.3033445831388235, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.044862336944788694, "step": 2593 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 860.8348693847656, "completions/mean_terminated_length": 729.5067749023438, "completions/min_length": 409.0, "completions/min_terminated_length": 409.0, "epoch": 0.7748487790306923, "grad_norm": 0.35708650946617126, "kl": 2.826171875, "learning_rate": 2.11318132869182e-06, "loss": 0.1422, "num_tokens": 1247477166.0, "reward": 0.745535746216774, "reward_std": 0.224678386002779, "rewards/accuracy_reward/mean": 0.2589285671710968, "rewards/accuracy_reward/std": 0.43614087998867035, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05488624423742294, "step": 2594 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 862.7812957763672, "completions/mean_terminated_length": 742.2123870849609, "completions/min_length": 289.75, "completions/min_terminated_length": 289.75, "epoch": 0.7751474871182137, "grad_norm": 0.28917258977890015, "kl": 2.18359375, "learning_rate": 2.106479578068501e-06, "loss": 0.1132, "num_tokens": 1247934316.0, "reward": 0.6847098618745804, "reward_std": 0.1496050152927637, "rewards/accuracy_reward/mean": 0.19642857322469354, "rewards/accuracy_reward/std": 0.34689511731266975, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.04951908718794584, "step": 2595 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1000.0, "completions/mean_length": 865.4129943847656, "completions/mean_terminated_length": 718.8236999511719, "completions/min_length": 297.25, "completions/min_terminated_length": 297.25, "epoch": 0.7754461952057352, "grad_norm": 0.36709392070770264, "kl": 1.763671875, "learning_rate": 2.0997872200373114e-06, "loss": 0.1074, "num_tokens": 1248393573.0, "reward": 0.6746651977300644, "reward_std": 0.15338828787207603, "rewards/accuracy_reward/mean": 0.18303571385331452, "rewards/accuracy_reward/std": 0.3422245793044567, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2596 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 836.560302734375, "completions/mean_terminated_length": 715.2419586181641, "completions/min_length": 341.25, "completions/min_terminated_length": 341.25, "epoch": 0.7757449032932566, "grad_norm": 0.2518214285373688, "kl": 1.71875, "learning_rate": 2.093104262561569e-06, "loss": 0.0989, "num_tokens": 1248839728.0, "reward": 0.6199776977300644, "reward_std": 0.15707704424858093, "rewards/accuracy_reward/mean": 0.12723214412108064, "rewards/accuracy_reward/std": 0.3074533976614475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 2597 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 884.3795013427734, "completions/mean_terminated_length": 763.8731689453125, "completions/min_length": 326.25, "completions/min_terminated_length": 326.25, "epoch": 0.7760436113807782, "grad_norm": 0.17108376324176788, "kl": 1.2900390625, "learning_rate": 2.086430713593397e-06, "loss": 0.0684, "num_tokens": 1249303610.0, "reward": 0.6021205484867096, "reward_std": 0.13797119003720582, "rewards/accuracy_reward/mean": 0.10714286006987095, "rewards/accuracy_reward/std": 0.2425001971423626, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 2598 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 891.0670013427734, "completions/mean_terminated_length": 774.1241912841797, "completions/min_length": 307.75, "completions/min_terminated_length": 307.75, "epoch": 0.7763423194682996, "grad_norm": 0.4708491861820221, "kl": 2.08203125, "learning_rate": 2.0797665810737386e-06, "loss": 0.1057, "num_tokens": 1249778984.0, "reward": 0.612723246216774, "reward_std": 0.14350335486233234, "rewards/accuracy_reward/mean": 0.12500000186264515, "rewards/accuracy_reward/std": 0.32295437529683113, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.0527555150911212, "step": 2599 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 865.216552734375, "completions/mean_terminated_length": 744.1906585693359, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "epoch": 0.7766410275558211, "grad_norm": 0.18530695140361786, "kl": 2.0390625, "learning_rate": 2.0731118729323164e-06, "loss": 0.1027, "num_tokens": 1250236361.0, "reward": 0.5485491305589676, "reward_std": 0.08578686090186238, "rewards/accuracy_reward/mean": 0.05803571455180645, "rewards/accuracy_reward/std": 0.19820642471313477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 2600 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 823.4955749511719, "completions/mean_terminated_length": 700.1464691162109, "completions/min_length": 290.5, "completions/min_terminated_length": 290.5, "epoch": 0.7769397356433425, "grad_norm": 0.2841646671295166, "kl": 2.92578125, "learning_rate": 2.0664665970876496e-06, "loss": 0.1579, "num_tokens": 1250672263.0, "reward": 0.6138393133878708, "reward_std": 0.12193071097135544, "rewards/accuracy_reward/mean": 0.12723213946446776, "rewards/accuracy_reward/std": 0.2978668734431267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.055819165892899036, "step": 2601 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 828.466552734375, "completions/mean_terminated_length": 691.6480560302734, "completions/min_length": 281.25, "completions/min_terminated_length": 281.25, "epoch": 0.777238443730864, "grad_norm": 0.6813576817512512, "kl": 2.298828125, "learning_rate": 2.059830761447025e-06, "loss": 0.1485, "num_tokens": 1251111832.0, "reward": 0.7087053954601288, "reward_std": 0.1552082933485508, "rewards/accuracy_reward/mean": 0.2269345223903656, "rewards/accuracy_reward/std": 0.41490666568279266, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04716797545552254, "step": 2602 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38169642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 859.0245819091797, "completions/mean_terminated_length": 752.9063720703125, "completions/min_length": 244.5, "completions/min_terminated_length": 244.5, "epoch": 0.7775371518183855, "grad_norm": 0.2129688411951065, "kl": 1.998046875, "learning_rate": 2.0532043739065054e-06, "loss": 0.1062, "num_tokens": 1251571283.0, "reward": 0.6277901977300644, "reward_std": 0.17947876453399658, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.34126149117946625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.048990159295499325, "step": 2603 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 886.1317291259766, "completions/mean_terminated_length": 780.4455108642578, "completions/min_length": 344.5, "completions/min_terminated_length": 344.5, "epoch": 0.777835859905907, "grad_norm": 0.2368520051240921, "kl": 1.798828125, "learning_rate": 2.046587442350901e-06, "loss": 0.0929, "num_tokens": 1252034942.0, "reward": 0.5708705633878708, "reward_std": 0.12692582700401545, "rewards/accuracy_reward/mean": 0.08891368936747313, "rewards/accuracy_reward/std": 0.22460028901696205, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 2604 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 883.341552734375, "completions/mean_terminated_length": 768.4949951171875, "completions/min_length": 304.75, "completions/min_terminated_length": 304.75, "epoch": 0.7781345679934284, "grad_norm": 0.2766672968864441, "kl": 1.98828125, "learning_rate": 2.0399799746537806e-06, "loss": 0.0948, "num_tokens": 1252505255.0, "reward": 0.6467634290456772, "reward_std": 0.17372108437120914, "rewards/accuracy_reward/mean": 0.159598208963871, "rewards/accuracy_reward/std": 0.3545946106314659, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047717904672026634, "step": 2605 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 837.5692291259766, "completions/mean_terminated_length": 700.7196807861328, "completions/min_length": 256.25, "completions/min_terminated_length": 256.25, "epoch": 0.7784332760809499, "grad_norm": 0.21773211658000946, "kl": 2.015625, "learning_rate": 2.0333819786774446e-06, "loss": 0.117, "num_tokens": 1252960582.0, "reward": 0.7606027275323868, "reward_std": 0.16641569882631302, "rewards/accuracy_reward/mean": 0.2678571380674839, "rewards/accuracy_reward/std": 0.423377089202404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.0412445142865181, "step": 2606 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 895.1607666015625, "completions/mean_terminated_length": 786.1817016601562, "completions/min_length": 378.75, "completions/min_terminated_length": 378.75, "epoch": 0.7787319841684713, "grad_norm": 0.29148349165916443, "kl": 2.21484375, "learning_rate": 2.02679346227293e-06, "loss": 0.107, "num_tokens": 1253427150.0, "reward": 0.6054687798023224, "reward_std": 0.1483269426971674, "rewards/accuracy_reward/mean": 0.11607143003493547, "rewards/accuracy_reward/std": 0.30430589616298676, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.0489901602268219, "step": 2607 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4977678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 881.310302734375, "completions/mean_terminated_length": 743.0610198974609, "completions/min_length": 363.75, "completions/min_terminated_length": 363.75, "epoch": 0.7790306922559929, "grad_norm": 0.27863624691963196, "kl": 2.25390625, "learning_rate": 2.0202144332799832e-06, "loss": 0.1066, "num_tokens": 1253900009.0, "reward": 0.6629464626312256, "reward_std": 0.14991804398596287, "rewards/accuracy_reward/mean": 0.17410714365541935, "rewards/accuracy_reward/std": 0.3754969611763954, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051120287738740444, "step": 2608 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 902.3482360839844, "completions/mean_terminated_length": 784.3817749023438, "completions/min_length": 411.25, "completions/min_terminated_length": 411.25, "epoch": 0.7793294003435143, "grad_norm": 0.2025088369846344, "kl": 1.392578125, "learning_rate": 2.013644899527074e-06, "loss": 0.0766, "num_tokens": 1254379909.0, "reward": 0.6417410969734192, "reward_std": 0.10874809697270393, "rewards/accuracy_reward/mean": 0.14732143096625805, "rewards/accuracy_reward/std": 0.3473358079791069, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03141601476818323, "step": 2609 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2857142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 782.4754791259766, "completions/mean_terminated_length": 685.3363189697266, "completions/min_length": 326.25, "completions/min_terminated_length": 326.25, "epoch": 0.7796281084310358, "grad_norm": 0.2693113684654236, "kl": 2.443359375, "learning_rate": 2.0070848688313603e-06, "loss": 0.1459, "num_tokens": 1254797882.0, "reward": 0.6372768208384514, "reward_std": 0.15981148835271597, "rewards/accuracy_reward/mean": 0.14732143096625805, "rewards/accuracy_reward/std": 0.2588426023721695, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 2610 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47321428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 881.0424346923828, "completions/mean_terminated_length": 751.3514404296875, "completions/min_length": 214.5, "completions/min_terminated_length": 214.5, "epoch": 0.7799268165185572, "grad_norm": 0.6621598601341248, "kl": 3.2734375, "learning_rate": 2.0005343489987038e-06, "loss": 0.152, "num_tokens": 1255268701.0, "reward": 0.6316964775323868, "reward_std": 0.20637012645602226, "rewards/accuracy_reward/mean": 0.14732142817229033, "rewards/accuracy_reward/std": 0.33064064756035805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4843749925494194, "rewards/tag_count_reward/std": 0.06053628120571375, "step": 2611 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 859.0491485595703, "completions/mean_terminated_length": 745.2859649658203, "completions/min_length": 295.75, "completions/min_terminated_length": 295.75, "epoch": 0.7802255246060787, "grad_norm": 0.3009810149669647, "kl": 2.21875, "learning_rate": 1.993993347823643e-06, "loss": 0.1332, "num_tokens": 1255723331.0, "reward": 0.6222098469734192, "reward_std": 0.15804456174373627, "rewards/accuracy_reward/mean": 0.13169642724096775, "rewards/accuracy_reward/std": 0.288053173571825, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 2612 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 885.7187805175781, "completions/mean_terminated_length": 787.7608337402344, "completions/min_length": 388.5, "completions/min_terminated_length": 388.5, "epoch": 0.7805242326936002, "grad_norm": 0.4042608141899109, "kl": 2.369140625, "learning_rate": 1.9874618730893947e-06, "loss": 0.1042, "num_tokens": 1256189637.0, "reward": 0.6428571492433548, "reward_std": 0.1602166760712862, "rewards/accuracy_reward/mean": 0.15401785960420966, "rewards/accuracy_reward/std": 0.3316444382071495, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05112028680741787, "step": 2613 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 865.4397735595703, "completions/mean_terminated_length": 761.6346282958984, "completions/min_length": 325.25, "completions/min_terminated_length": 325.25, "epoch": 0.7808229407811217, "grad_norm": 0.42125168442726135, "kl": 2.09765625, "learning_rate": 1.9809399325678326e-06, "loss": 0.0999, "num_tokens": 1256648890.0, "reward": 0.6930803805589676, "reward_std": 0.1492507979273796, "rewards/accuracy_reward/mean": 0.20089285261929035, "rewards/accuracy_reward/std": 0.38793106377124786, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 2614 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 861.1428985595703, "completions/mean_terminated_length": 764.9004974365234, "completions/min_length": 361.75, "completions/min_terminated_length": 361.75, "epoch": 0.7811216488686431, "grad_norm": 0.2911270558834076, "kl": 1.91015625, "learning_rate": 1.974427534019493e-06, "loss": 0.1074, "num_tokens": 1257105914.0, "reward": 0.6523437798023224, "reward_std": 0.1728758867830038, "rewards/accuracy_reward/mean": 0.16071428544819355, "rewards/accuracy_reward/std": 0.35967228561639786, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04293336346745491, "step": 2615 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 882.7567291259766, "completions/mean_terminated_length": 755.7010040283203, "completions/min_length": 332.5, "completions/min_terminated_length": 332.5, "epoch": 0.7814203569561646, "grad_norm": 0.21210987865924835, "kl": 2.35546875, "learning_rate": 1.967924685193552e-06, "loss": 0.1188, "num_tokens": 1257576877.0, "reward": 0.5318080633878708, "reward_std": 0.10938599146902561, "rewards/accuracy_reward/mean": 0.04352678614668548, "rewards/accuracy_reward/std": 0.18556606583297253, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 2616 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 868.2210083007812, "completions/mean_terminated_length": 741.0682983398438, "completions/min_length": 381.0, "completions/min_terminated_length": 381.0, "epoch": 0.781719065043686, "grad_norm": 0.42875176668167114, "kl": 2.55859375, "learning_rate": 1.961431393827827e-06, "loss": 0.1539, "num_tokens": 1258043024.0, "reward": 0.5982143133878708, "reward_std": 0.14429982751607895, "rewards/accuracy_reward/mean": 0.10937500116415322, "rewards/accuracy_reward/std": 0.28464323841035366, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05157411750406027, "step": 2617 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 819.7388763427734, "completions/mean_terminated_length": 725.4194030761719, "completions/min_length": 345.75, "completions/min_terminated_length": 345.75, "epoch": 0.7820177731312076, "grad_norm": 0.21410635113716125, "kl": 1.98828125, "learning_rate": 1.954947667648763e-06, "loss": 0.1115, "num_tokens": 1258476043.0, "reward": 0.746651828289032, "reward_std": 0.14391100406646729, "rewards/accuracy_reward/mean": 0.2544642873108387, "rewards/accuracy_reward/std": 0.4298102632164955, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2618 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 850.9598541259766, "completions/mean_terminated_length": 735.5638885498047, "completions/min_length": 205.0, "completions/min_terminated_length": 205.0, "epoch": 0.782316481218729, "grad_norm": 0.25715672969818115, "kl": 2.1640625, "learning_rate": 1.9484735143714184e-06, "loss": 0.1211, "num_tokens": 1258930233.0, "reward": 0.6445312723517418, "reward_std": 0.1349629545584321, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.2897576317191124, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 2619 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46428571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 846.9866485595703, "completions/mean_terminated_length": 700.3962554931641, "completions/min_length": 232.0, "completions/min_terminated_length": 232.0, "epoch": 0.7826151893062505, "grad_norm": 0.41352924704551697, "kl": 3.33203125, "learning_rate": 1.942008941699465e-06, "loss": 0.1728, "num_tokens": 1259385939.0, "reward": 0.5474330633878708, "reward_std": 0.10705520398914814, "rewards/accuracy_reward/mean": 0.06473214295692742, "rewards/accuracy_reward/std": 0.22568520717322826, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06349284294992685, "step": 2620 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 827.9844055175781, "completions/mean_terminated_length": 710.9698486328125, "completions/min_length": 331.5, "completions/min_terminated_length": 331.5, "epoch": 0.7829138973937719, "grad_norm": 0.3871164321899414, "kl": 2.33203125, "learning_rate": 1.9355539573251737e-06, "loss": 0.1205, "num_tokens": 1259830444.0, "reward": 0.7321428880095482, "reward_std": 0.16937563754618168, "rewards/accuracy_reward/mean": 0.2433035783469677, "rewards/accuracy_reward/std": 0.3263813406229019, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 2621 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33705357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 808.3415374755859, "completions/mean_terminated_length": 700.3542938232422, "completions/min_length": 211.25, "completions/min_terminated_length": 211.25, "epoch": 0.7832126054812935, "grad_norm": 0.2216871976852417, "kl": 2.3125, "learning_rate": 1.9291085689294074e-06, "loss": 0.1257, "num_tokens": 1260255557.0, "reward": 0.7243303805589676, "reward_std": 0.14089334290474653, "rewards/accuracy_reward/mean": 0.240699402987957, "rewards/accuracy_reward/std": 0.4161013141274452, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2622 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 877.5491485595703, "completions/mean_terminated_length": 769.117919921875, "completions/min_length": 405.0, "completions/min_terminated_length": 405.0, "epoch": 0.7835113135688149, "grad_norm": 0.24185515940189362, "kl": 1.404296875, "learning_rate": 1.922672784181605e-06, "loss": 0.0683, "num_tokens": 1260724795.0, "reward": 0.555245578289032, "reward_std": 0.11550952028483152, "rewards/accuracy_reward/mean": 0.06249999930150807, "rewards/accuracy_reward/std": 0.22179833613336086, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 2623 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 828.8348541259766, "completions/mean_terminated_length": 739.2461395263672, "completions/min_length": 327.0, "completions/min_terminated_length": 327.0, "epoch": 0.7838100216563364, "grad_norm": 0.3279132843017578, "kl": 1.84765625, "learning_rate": 1.916246610739787e-06, "loss": 0.1042, "num_tokens": 1261168817.0, "reward": 0.7237723618745804, "reward_std": 0.18063464760780334, "rewards/accuracy_reward/mean": 0.2321428544819355, "rewards/accuracy_reward/std": 0.40998557209968567, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 2624 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 776.2210235595703, "completions/mean_terminated_length": 674.6158599853516, "completions/min_length": 333.75, "completions/min_terminated_length": 333.75, "epoch": 0.7841087297438578, "grad_norm": 0.6046783924102783, "kl": 1.810546875, "learning_rate": 1.9098300562505266e-06, "loss": 0.1092, "num_tokens": 1261592180.0, "reward": 0.8331473618745804, "reward_std": 0.11630581691861153, "rewards/accuracy_reward/mean": 0.3415178544819355, "rewards/accuracy_reward/std": 0.46552715450525284, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2625 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 858.8192443847656, "completions/mean_terminated_length": 752.2846374511719, "completions/min_length": 403.25, "completions/min_terminated_length": 403.25, "epoch": 0.7844074378313793, "grad_norm": 0.5230125188827515, "kl": 2.4140625, "learning_rate": 1.903423128348959e-06, "loss": 0.1262, "num_tokens": 1262052739.0, "reward": 0.6551339477300644, "reward_std": 0.1426073219627142, "rewards/accuracy_reward/mean": 0.17894344893284142, "rewards/accuracy_reward/std": 0.3354004807770252, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053606295958161354, "step": 2626 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.75, "completions/mean_length": 823.4330749511719, "completions/mean_terminated_length": 717.4292144775391, "completions/min_length": 338.0, "completions/min_terminated_length": 338.0, "epoch": 0.7847061459189008, "grad_norm": 0.31206148862838745, "kl": 2.740234375, "learning_rate": 1.8970258346587645e-06, "loss": 0.1669, "num_tokens": 1262499189.0, "reward": 0.6830357536673546, "reward_std": 0.1797340949997306, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.3224512189626694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050612835213541985, "step": 2627 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 862.3259429931641, "completions/mean_terminated_length": 741.5764007568359, "completions/min_length": 352.0, "completions/min_terminated_length": 352.0, "epoch": 0.7850048540064222, "grad_norm": 0.2800358831882477, "kl": 2.533203125, "learning_rate": 1.8906381827921583e-06, "loss": 0.1367, "num_tokens": 1262959447.0, "reward": 0.6914062798023224, "reward_std": 0.12922626174986362, "rewards/accuracy_reward/mean": 0.20312499860301614, "rewards/accuracy_reward/std": 0.364499744027853, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05133028235286474, "step": 2628 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 844.4107513427734, "completions/mean_terminated_length": 751.3423919677734, "completions/min_length": 208.5, "completions/min_terminated_length": 208.5, "epoch": 0.7853035620939437, "grad_norm": 0.27534589171409607, "kl": 2.771484375, "learning_rate": 1.8842601803498772e-06, "loss": 0.1445, "num_tokens": 1263413631.0, "reward": 0.6026785969734192, "reward_std": 0.14975177869200706, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.2438780590891838, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05384558532387018, "step": 2629 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28794642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 808.6853179931641, "completions/mean_terminated_length": 721.8948516845703, "completions/min_length": 256.5, "completions/min_terminated_length": 256.5, "epoch": 0.7856022701814651, "grad_norm": 0.27039241790771484, "kl": 2.1953125, "learning_rate": 1.877891834921186e-06, "loss": 0.1181, "num_tokens": 1263852754.0, "reward": 0.6847098469734192, "reward_std": 0.16682086139917374, "rewards/accuracy_reward/mean": 0.19196428847499192, "rewards/accuracy_reward/std": 0.3345234375447035, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 2630 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37499999999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 831.8370819091797, "completions/mean_terminated_length": 726.3886871337891, "completions/min_length": 387.25, "completions/min_terminated_length": 387.25, "epoch": 0.7859009782689866, "grad_norm": 0.27063125371932983, "kl": 3.05859375, "learning_rate": 1.8715331540838488e-06, "loss": 0.1893, "num_tokens": 1264290697.0, "reward": 0.7232143133878708, "reward_std": 0.18822535127401352, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.4134906008839607, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 2631 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3370535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 831.841552734375, "completions/mean_terminated_length": 743.2537384033203, "completions/min_length": 324.25, "completions/min_terminated_length": 324.25, "epoch": 0.786199686356508, "grad_norm": 0.5330144762992859, "kl": 2.51171875, "learning_rate": 1.8651841454041376e-06, "loss": 0.1261, "num_tokens": 1264738322.0, "reward": 0.6445312798023224, "reward_std": 0.15961035154759884, "rewards/accuracy_reward/mean": 0.1540178586728871, "rewards/accuracy_reward/std": 0.3244112767279148, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04582912987098098, "step": 2632 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 823.1406555175781, "completions/mean_terminated_length": 720.2854156494141, "completions/min_length": 310.75, "completions/min_terminated_length": 310.75, "epoch": 0.7864983944440296, "grad_norm": 0.3356974422931671, "kl": 2.70703125, "learning_rate": 1.858844816436809e-06, "loss": 0.1507, "num_tokens": 1265175937.0, "reward": 0.6908482313156128, "reward_std": 0.2298361249268055, "rewards/accuracy_reward/mean": 0.2008928544819355, "rewards/accuracy_reward/std": 0.39811505377292633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 2633 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28348214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 789.3504638671875, "completions/mean_terminated_length": 699.8630981445312, "completions/min_length": 265.25, "completions/min_terminated_length": 265.25, "epoch": 0.786797102531551, "grad_norm": 0.4289930462837219, "kl": 2.7890625, "learning_rate": 1.8525151747251058e-06, "loss": 0.1371, "num_tokens": 1265603374.0, "reward": 0.675223246216774, "reward_std": 0.14362751133739948, "rewards/accuracy_reward/mean": 0.1904761902987957, "rewards/accuracy_reward/std": 0.3748863935470581, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 2634 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 781.0379791259766, "completions/mean_terminated_length": 673.4861450195312, "completions/min_length": 274.75, "completions/min_terminated_length": 274.75, "epoch": 0.7870958106190725, "grad_norm": 0.5955332517623901, "kl": 3.765625, "learning_rate": 1.8461952278007434e-06, "loss": 0.1985, "num_tokens": 1266024319.0, "reward": 0.6322544813156128, "reward_std": 0.14952012710273266, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.34292856976389885, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05868698563426733, "step": 2635 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 799.5402069091797, "completions/mean_terminated_length": 699.8459930419922, "completions/min_length": 299.0, "completions/min_terminated_length": 299.0, "epoch": 0.7873945187065939, "grad_norm": 0.31727951765060425, "kl": 2.556640625, "learning_rate": 1.8398849831839017e-06, "loss": 0.1528, "num_tokens": 1266449329.0, "reward": 0.678013414144516, "reward_std": 0.15620629116892815, "rewards/accuracy_reward/mean": 0.18750000116415322, "rewards/accuracy_reward/std": 0.3462738152593374, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047210452146828175, "step": 2636 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2745535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 799.3995819091797, "completions/mean_terminated_length": 714.802978515625, "completions/min_length": 378.0, "completions/min_terminated_length": 378.0, "epoch": 0.7876932267941155, "grad_norm": 0.27497565746307373, "kl": 2.076171875, "learning_rate": 1.833584448383211e-06, "loss": 0.1219, "num_tokens": 1266883796.0, "reward": 0.690848246216774, "reward_std": 0.15526974946260452, "rewards/accuracy_reward/mean": 0.19866071362048388, "rewards/accuracy_reward/std": 0.3459206484258175, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.04197291610762477, "step": 2637 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 863.419677734375, "completions/mean_terminated_length": 759.785400390625, "completions/min_length": 353.5, "completions/min_terminated_length": 353.5, "epoch": 0.7879919348816369, "grad_norm": 0.33534860610961914, "kl": 1.5546875, "learning_rate": 1.8272936308957556e-06, "loss": 0.0728, "num_tokens": 1267346016.0, "reward": 0.6830357611179352, "reward_std": 0.18781456165015697, "rewards/accuracy_reward/mean": 0.18973214086145163, "rewards/accuracy_reward/std": 0.37161362171173096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 2638 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714288, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.5, "completions/mean_length": 769.0335083007812, "completions/mean_terminated_length": 688.5743560791016, "completions/min_length": 367.5, "completions/min_terminated_length": 367.5, "epoch": 0.7882906429691584, "grad_norm": 0.2655164897441864, "kl": 2.16015625, "learning_rate": 1.8210125382070466e-06, "loss": 0.1385, "num_tokens": 1267760031.0, "reward": 0.6735491305589676, "reward_std": 0.13425388792529702, "rewards/accuracy_reward/mean": 0.1808035708963871, "rewards/accuracy_reward/std": 0.3097771182656288, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 2639 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 807.6138610839844, "completions/mean_terminated_length": 719.4970855712891, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.7885893510566798, "grad_norm": 0.2605459988117218, "kl": 2.251953125, "learning_rate": 1.814741177791034e-06, "loss": 0.1197, "num_tokens": 1268192402.0, "reward": 0.7667411118745804, "reward_std": 0.20979386195540428, "rewards/accuracy_reward/mean": 0.2767857164144516, "rewards/accuracy_reward/std": 0.44886061549186707, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04994574096053839, "step": 2640 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.296875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 821.5580749511719, "completions/mean_terminated_length": 735.7080230712891, "completions/min_length": 230.5, "completions/min_terminated_length": 230.5, "epoch": 0.7888880591442013, "grad_norm": 0.37195560336112976, "kl": 2.4609375, "learning_rate": 1.808479557110081e-06, "loss": 0.1518, "num_tokens": 1268632508.0, "reward": 0.7042410969734192, "reward_std": 0.18078800290822983, "rewards/accuracy_reward/mean": 0.21428571827709675, "rewards/accuracy_reward/std": 0.3995094373822212, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 2641 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2566964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.25, "completions/mean_length": 782.6585388183594, "completions/mean_terminated_length": 699.8766021728516, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "epoch": 0.7891867672317228, "grad_norm": 0.4665696918964386, "kl": 2.513671875, "learning_rate": 1.8022276836149678e-06, "loss": 0.1458, "num_tokens": 1269062355.0, "reward": 0.6992187798023224, "reward_std": 0.15548510290682316, "rewards/accuracy_reward/mean": 0.20982142630964518, "rewards/accuracy_reward/std": 0.3704650327563286, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2642 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29464285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 813.4464721679688, "completions/mean_terminated_length": 724.3894500732422, "completions/min_length": 315.5, "completions/min_terminated_length": 315.5, "epoch": 0.7894854753192443, "grad_norm": 0.25426799058914185, "kl": 2.548828125, "learning_rate": 1.7959855647448642e-06, "loss": 0.1222, "num_tokens": 1269502107.0, "reward": 0.5507812798023224, "reward_std": 0.10092113073915243, "rewards/accuracy_reward/mean": 0.06026785634458065, "rewards/accuracy_reward/std": 0.16348886489868164, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047210452146828175, "step": 2643 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 822.3303833007812, "completions/mean_terminated_length": 720.5358123779297, "completions/min_length": 261.75, "completions/min_terminated_length": 261.75, "epoch": 0.7897841834067657, "grad_norm": 0.33838722109794617, "kl": 2.98046875, "learning_rate": 1.7897532079273471e-06, "loss": 0.1578, "num_tokens": 1269939279.0, "reward": 0.6646205633878708, "reward_std": 0.1418316848576069, "rewards/accuracy_reward/mean": 0.17857142770662904, "rewards/accuracy_reward/std": 0.3407112769782543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05534125491976738, "step": 2644 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32589285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 994.75, "completions/mean_length": 791.4777221679688, "completions/mean_terminated_length": 682.2559661865234, "completions/min_length": 216.5, "completions/min_terminated_length": 216.5, "epoch": 0.7900828914942872, "grad_norm": 0.3601532578468323, "kl": 1.7626953125, "learning_rate": 1.7835306205783643e-06, "loss": 0.0928, "num_tokens": 1270364517.0, "reward": 0.7031250447034836, "reward_std": 0.09565623663365841, "rewards/accuracy_reward/mean": 0.2098214253783226, "rewards/accuracy_reward/std": 0.3878720924258232, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03914389340206981, "step": 2645 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 835.5469055175781, "completions/mean_terminated_length": 740.3494415283203, "completions/min_length": 269.5, "completions/min_terminated_length": 269.5, "epoch": 0.7903815995818086, "grad_norm": 0.5285109877586365, "kl": 1.869140625, "learning_rate": 1.7773178101022514e-06, "loss": 0.1185, "num_tokens": 1270809338.0, "reward": 0.6177455633878708, "reward_std": 0.13208701089024544, "rewards/accuracy_reward/mean": 0.12499999813735485, "rewards/accuracy_reward/std": 0.2546691820025444, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2646 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 882.3683471679688, "completions/mean_terminated_length": 775.8009033203125, "completions/min_length": 332.25, "completions/min_terminated_length": 332.25, "epoch": 0.7906803076693302, "grad_norm": 0.14713144302368164, "kl": 1.3369140625, "learning_rate": 1.7711147838916987e-06, "loss": 0.0622, "num_tokens": 1271274735.0, "reward": 0.6774553954601288, "reward_std": 0.09548172913491726, "rewards/accuracy_reward/mean": 0.1830357131548226, "rewards/accuracy_reward/std": 0.3377839885652065, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196492433548, "rewards/tag_count_reward/std": 0.034913196228444576, "step": 2647 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 801.1451110839844, "completions/mean_terminated_length": 704.7887268066406, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "epoch": 0.7909790157568516, "grad_norm": 0.3663354218006134, "kl": 2.048828125, "learning_rate": 1.7649215493277617e-06, "loss": 0.1402, "num_tokens": 1271705520.0, "reward": 0.8074777126312256, "reward_std": 0.2555457651615143, "rewards/accuracy_reward/mean": 0.3147321417927742, "rewards/accuracy_reward/std": 0.46592526137828827, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.042059858329594135, "step": 2648 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3058035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 779.3036041259766, "completions/mean_terminated_length": 669.6910247802734, "completions/min_length": 205.75, "completions/min_terminated_length": 205.75, "epoch": 0.7912777238443731, "grad_norm": 0.4501935839653015, "kl": 2.5078125, "learning_rate": 1.7587381137798432e-06, "loss": 0.1444, "num_tokens": 1272129960.0, "reward": 0.722098246216774, "reward_std": 0.18151237815618515, "rewards/accuracy_reward/mean": 0.2321428544819355, "rewards/accuracy_reward/std": 0.398986779153347, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 2649 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 852.4955596923828, "completions/mean_terminated_length": 737.1662139892578, "completions/min_length": 371.5, "completions/min_terminated_length": 371.5, "epoch": 0.7915764319318945, "grad_norm": 0.36230671405792236, "kl": 1.61328125, "learning_rate": 1.7525644846056877e-06, "loss": 0.0961, "num_tokens": 1272582342.0, "reward": 0.7070312798023224, "reward_std": 0.18497493490576744, "rewards/accuracy_reward/mean": 0.2194940447807312, "rewards/accuracy_reward/std": 0.4025406688451767, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04124451335519552, "step": 2650 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 820.8281555175781, "completions/mean_terminated_length": 721.6601104736328, "completions/min_length": 281.5, "completions/min_terminated_length": 281.5, "epoch": 0.7918751400194161, "grad_norm": 0.2833208441734314, "kl": 2.5859375, "learning_rate": 1.7464006691513624e-06, "loss": 0.1353, "num_tokens": 1273017737.0, "reward": 0.7120536118745804, "reward_std": 0.11956499144434929, "rewards/accuracy_reward/mean": 0.2209821417927742, "rewards/accuracy_reward/std": 0.396023016422987, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194884091616, "step": 2651 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.75, "completions/mean_length": 814.6897735595703, "completions/mean_terminated_length": 697.8232879638672, "completions/min_length": 319.5, "completions/min_terminated_length": 319.5, "epoch": 0.7921738481069375, "grad_norm": 0.37025564908981323, "kl": 2.427734375, "learning_rate": 1.7402466747512704e-06, "loss": 0.1344, "num_tokens": 1273451198.0, "reward": 0.6718750298023224, "reward_std": 0.20535986125469208, "rewards/accuracy_reward/mean": 0.18080356996506453, "rewards/accuracy_reward/std": 0.36642102897167206, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194884091616, "step": 2652 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45758928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 877.2076263427734, "completions/mean_terminated_length": 752.7817993164062, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "epoch": 0.792472556194459, "grad_norm": 0.2705496847629547, "kl": 2.287109375, "learning_rate": 1.7341025087281149e-06, "loss": 0.1127, "num_tokens": 1273919643.0, "reward": 0.597098246216774, "reward_std": 0.11649600695818663, "rewards/accuracy_reward/mean": 0.10714286006987095, "rewards/accuracy_reward/std": 0.25758620351552963, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2653 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32812500000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 818.7567443847656, "completions/mean_terminated_length": 717.7623138427734, "completions/min_length": 343.0, "completions/min_terminated_length": 343.0, "epoch": 0.7927712642819804, "grad_norm": 0.3022988438606262, "kl": 3.26171875, "learning_rate": 1.7279681783929136e-06, "loss": 0.1689, "num_tokens": 1274357054.0, "reward": 0.569196455180645, "reward_std": 0.11790806427598, "rewards/accuracy_reward/mean": 0.0825892873108387, "rewards/accuracy_reward/std": 0.23171361908316612, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05643500294536352, "step": 2654 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 808.9442443847656, "completions/mean_terminated_length": 698.0382995605469, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "epoch": 0.7930699723695019, "grad_norm": 0.4146200716495514, "kl": 2.162109375, "learning_rate": 1.7218436910449787e-06, "loss": 0.1188, "num_tokens": 1274787045.0, "reward": 0.6796875298023224, "reward_std": 0.20676103606820107, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3857983648777008, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 2655 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 817.0245971679688, "completions/mean_terminated_length": 707.7773742675781, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.7933686804570234, "grad_norm": 0.3875057101249695, "kl": 2.67578125, "learning_rate": 1.7157290539719108e-06, "loss": 0.157, "num_tokens": 1275225584.0, "reward": 0.6824776977300644, "reward_std": 0.1517825424671173, "rewards/accuracy_reward/mean": 0.19196428544819355, "rewards/accuracy_reward/std": 0.39023640751838684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04534579161554575, "step": 2656 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 848.200927734375, "completions/mean_terminated_length": 723.2505187988281, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 0.7936673885445449, "grad_norm": 0.40393272042274475, "kl": 2.369140625, "learning_rate": 1.709624274449584e-06, "loss": 0.1223, "num_tokens": 1275677194.0, "reward": 0.6032366305589676, "reward_std": 0.1503256242722273, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.3083554804325104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.489397332072258, "rewards/tag_count_reward/std": 0.050546927377581596, "step": 2657 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 848.8259124755859, "completions/mean_terminated_length": 732.5060577392578, "completions/min_length": 420.75, "completions/min_terminated_length": 420.75, "epoch": 0.7939660966320663, "grad_norm": 0.345624178647995, "kl": 3.62890625, "learning_rate": 1.70352935974215e-06, "loss": 0.1903, "num_tokens": 1276126460.0, "reward": 0.666294664144516, "reward_std": 0.16052201390266418, "rewards/accuracy_reward/mean": 0.18303571362048388, "rewards/accuracy_reward/std": 0.3409705422818661, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589328289032, "rewards/tag_count_reward/std": 0.06120228487998247, "step": 2658 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 853.3817443847656, "completions/mean_terminated_length": 726.1347351074219, "completions/min_length": 324.25, "completions/min_terminated_length": 324.25, "epoch": 0.7942648047195878, "grad_norm": 0.27873480319976807, "kl": 2.984375, "learning_rate": 1.697444317102015e-06, "loss": 0.1558, "num_tokens": 1276583703.0, "reward": 0.7237723618745804, "reward_std": 0.1420580893754959, "rewards/accuracy_reward/mean": 0.2366071417927742, "rewards/accuracy_reward/std": 0.41313622146844864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05430487543344498, "step": 2659 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 854.3013763427734, "completions/mean_terminated_length": 724.6897888183594, "completions/min_length": 336.0, "completions/min_terminated_length": 336.0, "epoch": 0.7945635128071092, "grad_norm": 0.3198690116405487, "kl": 3.66015625, "learning_rate": 1.6913691537698473e-06, "loss": 0.1999, "num_tokens": 1277042094.0, "reward": 0.654575914144516, "reward_std": 0.1779150702059269, "rewards/accuracy_reward/mean": 0.16964285587891936, "rewards/accuracy_reward/std": 0.3413065895438194, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.058686986565589905, "step": 2660 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 807.5558319091797, "completions/mean_terminated_length": 701.7468109130859, "completions/min_length": 275.5, "completions/min_terminated_length": 275.5, "epoch": 0.7948622208946308, "grad_norm": 0.4768582880496979, "kl": 1.9326171875, "learning_rate": 1.6853038769745466e-06, "loss": 0.092, "num_tokens": 1277481319.0, "reward": 0.7020089626312256, "reward_std": 0.2054751254618168, "rewards/accuracy_reward/mean": 0.2098214291036129, "rewards/accuracy_reward/std": 0.40400610119104385, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.046088166534900665, "step": 2661 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.0, "completions/mean_length": 849.1897735595703, "completions/mean_terminated_length": 742.3549041748047, "completions/min_length": 288.75, "completions/min_terminated_length": 288.75, "epoch": 0.7951609289821522, "grad_norm": 0.27622106671333313, "kl": 2.3671875, "learning_rate": 1.6792484939332653e-06, "loss": 0.1331, "num_tokens": 1277936060.0, "reward": 0.6696428954601288, "reward_std": 0.1768322754651308, "rewards/accuracy_reward/mean": 0.17857142724096775, "rewards/accuracy_reward/std": 0.3802775964140892, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589571990072727, "step": 2662 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34598214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 802.5268096923828, "completions/mean_terminated_length": 688.1565093994141, "completions/min_length": 246.75, "completions/min_terminated_length": 246.75, "epoch": 0.7954596370696737, "grad_norm": 0.7009884119033813, "kl": 2.630859375, "learning_rate": 1.6732030118513741e-06, "loss": 0.1554, "num_tokens": 1278362424.0, "reward": 0.7751116305589676, "reward_std": 0.16809895262122154, "rewards/accuracy_reward/mean": 0.285714291036129, "rewards/accuracy_reward/std": 0.4152417667210102, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2663 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 845.4553833007812, "completions/mean_terminated_length": 739.7826232910156, "completions/min_length": 372.75, "completions/min_terminated_length": 372.75, "epoch": 0.7957583451571951, "grad_norm": 0.3017551600933075, "kl": 3.052734375, "learning_rate": 1.6671674379224568e-06, "loss": 0.1502, "num_tokens": 1278811428.0, "reward": 0.607700914144516, "reward_std": 0.17424213886260986, "rewards/accuracy_reward/mean": 0.12053571455180645, "rewards/accuracy_reward/std": 0.29540836066007614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05382578261196613, "step": 2664 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 826.1049499511719, "completions/mean_terminated_length": 728.2837066650391, "completions/min_length": 333.5, "completions/min_terminated_length": 333.5, "epoch": 0.7960570532447167, "grad_norm": 0.3712005615234375, "kl": 1.470703125, "learning_rate": 1.6611417793283192e-06, "loss": 0.088, "num_tokens": 1279265475.0, "reward": 0.6668527126312256, "reward_std": 0.15486954897642136, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.31276045739650726, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03449268685653806, "step": 2665 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 826.669677734375, "completions/mean_terminated_length": 716.0399932861328, "completions/min_length": 311.75, "completions/min_terminated_length": 311.75, "epoch": 0.7963557613322381, "grad_norm": 0.46107664704322815, "kl": 2.4296875, "learning_rate": 1.655126043238957e-06, "loss": 0.1358, "num_tokens": 1279716607.0, "reward": 0.6378348469734192, "reward_std": 0.14975773356854916, "rewards/accuracy_reward/mean": 0.1473214274737984, "rewards/accuracy_reward/std": 0.291381923481822, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 2666 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 850.3705902099609, "completions/mean_terminated_length": 743.2344818115234, "completions/min_length": 339.75, "completions/min_terminated_length": 339.75, "epoch": 0.7966544694197596, "grad_norm": 0.3591119945049286, "kl": 2.51953125, "learning_rate": 1.6491202368125703e-06, "loss": 0.1262, "num_tokens": 1280165925.0, "reward": 0.6071428805589676, "reward_std": 0.2009751945734024, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.3221058249473572, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 2667 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 858.4174499511719, "completions/mean_terminated_length": 763.4148406982422, "completions/min_length": 360.75, "completions/min_terminated_length": 360.75, "epoch": 0.796953177507281, "grad_norm": 0.20535127818584442, "kl": 1.8046875, "learning_rate": 1.6431243671955344e-06, "loss": 0.0949, "num_tokens": 1280620496.0, "reward": 0.5898437798023224, "reward_std": 0.13220198266208172, "rewards/accuracy_reward/mean": 0.09821428544819355, "rewards/accuracy_reward/std": 0.28217749670147896, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2668 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.0, "completions/mean_length": 808.966552734375, "completions/mean_terminated_length": 685.8210754394531, "completions/min_length": 310.5, "completions/min_terminated_length": 310.5, "epoch": 0.7972518855948025, "grad_norm": 0.2574824392795563, "kl": 2.5, "learning_rate": 1.6371384415224046e-06, "loss": 0.147, "num_tokens": 1281053601.0, "reward": 0.6741071790456772, "reward_std": 0.1897001750767231, "rewards/accuracy_reward/mean": 0.18303571082651615, "rewards/accuracy_reward/std": 0.37314169853925705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04444765392690897, "step": 2669 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.30580357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 795.7768249511719, "completions/mean_terminated_length": 695.2750091552734, "completions/min_length": 356.5, "completions/min_terminated_length": 356.5, "epoch": 0.797550593682324, "grad_norm": 0.5316707491874695, "kl": 2.537109375, "learning_rate": 1.6311624669159064e-06, "loss": 0.1455, "num_tokens": 1281482941.0, "reward": 0.6623884290456772, "reward_std": 0.13952075690031052, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.2872391603887081, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047066682018339634, "step": 2670 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 816.7656555175781, "completions/mean_terminated_length": 720.5549011230469, "completions/min_length": 434.25, "completions/min_terminated_length": 434.25, "epoch": 0.7978493017698454, "grad_norm": 0.22843517363071442, "kl": 2.4375, "learning_rate": 1.6251964504869221e-06, "loss": 0.1346, "num_tokens": 1281920628.0, "reward": 0.6266741380095482, "reward_std": 0.1275594374164939, "rewards/accuracy_reward/mean": 0.13616071455180645, "rewards/accuracy_reward/std": 0.27811249345541, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 2671 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2633928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 791.3393249511719, "completions/mean_terminated_length": 710.3567962646484, "completions/min_length": 306.75, "completions/min_terminated_length": 306.75, "epoch": 0.7981480098573669, "grad_norm": 0.5473888516426086, "kl": 2.896484375, "learning_rate": 1.6192403993344808e-06, "loss": 0.1539, "num_tokens": 1282348780.0, "reward": 0.6635044813156128, "reward_std": 0.1436657067388296, "rewards/accuracy_reward/mean": 0.17410714086145163, "rewards/accuracy_reward/std": 0.34710701555013657, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04239992145448923, "step": 2672 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 816.8504791259766, "completions/mean_terminated_length": 730.9503631591797, "completions/min_length": 332.75, "completions/min_terminated_length": 332.75, "epoch": 0.7984467179448883, "grad_norm": 0.208663210272789, "kl": 2.396484375, "learning_rate": 1.6132943205457607e-06, "loss": 0.1267, "num_tokens": 1282789769.0, "reward": 0.5870536118745804, "reward_std": 0.1345380451530218, "rewards/accuracy_reward/mean": 0.09598214505240321, "rewards/accuracy_reward/std": 0.28308118134737015, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 2673 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.25, "completions/mean_length": 796.7991333007812, "completions/mean_terminated_length": 669.5233001708984, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "epoch": 0.7987454260324098, "grad_norm": 0.43150782585144043, "kl": 3.8359375, "learning_rate": 1.607358221196068e-06, "loss": 0.2091, "num_tokens": 1283216959.0, "reward": 0.8565848618745804, "reward_std": 0.2516101934015751, "rewards/accuracy_reward/mean": 0.3727678507566452, "rewards/accuracy_reward/std": 0.4473213851451874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.483816958963871, "rewards/tag_count_reward/std": 0.06030225474387407, "step": 2674 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38616071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 833.107177734375, "completions/mean_terminated_length": 712.3106842041016, "completions/min_length": 346.5, "completions/min_terminated_length": 346.5, "epoch": 0.7990441341199312, "grad_norm": 0.3279425799846649, "kl": 2.59375, "learning_rate": 1.6014321083488372e-06, "loss": 0.1613, "num_tokens": 1283658911.0, "reward": 0.6188616156578064, "reward_std": 0.20898422226309776, "rewards/accuracy_reward/mean": 0.13095237873494625, "rewards/accuracy_reward/std": 0.33004244416952133, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 2675 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 849.9665374755859, "completions/mean_terminated_length": 722.2251892089844, "completions/min_length": 289.5, "completions/min_terminated_length": 289.5, "epoch": 0.7993428422074528, "grad_norm": 0.20040911436080933, "kl": 2.236328125, "learning_rate": 1.5955159890556182e-06, "loss": 0.1242, "num_tokens": 1284112032.0, "reward": 0.606026828289032, "reward_std": 0.15055852755904198, "rewards/accuracy_reward/mean": 0.11607142677530646, "rewards/accuracy_reward/std": 0.3035138249397278, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 2676 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 812.607177734375, "completions/mean_terminated_length": 710.0411682128906, "completions/min_length": 256.0, "completions/min_terminated_length": 256.0, "epoch": 0.7996415502949742, "grad_norm": 0.4340626299381256, "kl": 2.423828125, "learning_rate": 1.589609870356076e-06, "loss": 0.1292, "num_tokens": 1284551456.0, "reward": 0.5943080484867096, "reward_std": 0.14745560474693775, "rewards/accuracy_reward/mean": 0.10267857229337096, "rewards/accuracy_reward/std": 0.2808152884244919, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 2677 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 816.7902069091797, "completions/mean_terminated_length": 716.9135437011719, "completions/min_length": 359.75, "completions/min_terminated_length": 359.75, "epoch": 0.7999402583824957, "grad_norm": 0.5872063636779785, "kl": 2.6015625, "learning_rate": 1.5837137592779628e-06, "loss": 0.1388, "num_tokens": 1284994098.0, "reward": 0.6568080633878708, "reward_std": 0.20029954984784126, "rewards/accuracy_reward/mean": 0.1651785671710968, "rewards/accuracy_reward/std": 0.34554677084088326, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04288960574194789, "step": 2678 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 818.3973541259766, "completions/mean_terminated_length": 716.2872619628906, "completions/min_length": 233.0, "completions/min_terminated_length": 233.0, "epoch": 0.8002389664700171, "grad_norm": 0.4305047392845154, "kl": 2.73828125, "learning_rate": 1.577827662837136e-06, "loss": 0.1389, "num_tokens": 1285434180.0, "reward": 0.6227678954601288, "reward_std": 0.1362494695931673, "rewards/accuracy_reward/mean": 0.13392856949940324, "rewards/accuracy_reward/std": 0.2532048113644123, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 2679 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2901785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 799.4219055175781, "completions/mean_terminated_length": 712.2356414794922, "completions/min_length": 262.5, "completions/min_terminated_length": 262.5, "epoch": 0.8005376745575387, "grad_norm": 0.3452804386615753, "kl": 1.748046875, "learning_rate": 1.5719515880375247e-06, "loss": 0.0978, "num_tokens": 1285869025.0, "reward": 0.7131696790456772, "reward_std": 0.13817952387034893, "rewards/accuracy_reward/mean": 0.21874999813735485, "rewards/accuracy_reward/std": 0.3983088955283165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 2680 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 841.9330596923828, "completions/mean_terminated_length": 728.3957824707031, "completions/min_length": 340.5, "completions/min_terminated_length": 340.5, "epoch": 0.8008363826450601, "grad_norm": 0.352495402097702, "kl": 2.630859375, "learning_rate": 1.566085541871145e-06, "loss": 0.1301, "num_tokens": 1286328195.0, "reward": 0.5797991305589676, "reward_std": 0.13735275156795979, "rewards/accuracy_reward/mean": 0.08928571408614516, "rewards/accuracy_reward/std": 0.27187687531113625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047917463816702366, "step": 2681 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3102678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 816.0669860839844, "completions/mean_terminated_length": 722.4334869384766, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "epoch": 0.8011350907325816, "grad_norm": 0.3180195093154907, "kl": 2.36328125, "learning_rate": 1.5602295313180683e-06, "loss": 0.1299, "num_tokens": 1286762305.0, "reward": 0.7008928954601288, "reward_std": 0.15580398216843605, "rewards/accuracy_reward/mean": 0.20982142724096775, "rewards/accuracy_reward/std": 0.39058130234479904, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.0447555473074317, "step": 2682 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 812.2745971679688, "completions/mean_terminated_length": 717.4511413574219, "completions/min_length": 270.25, "completions/min_terminated_length": 270.25, "epoch": 0.801433798820103, "grad_norm": 0.369143545627594, "kl": 2.82421875, "learning_rate": 1.5543835633464321e-06, "loss": 0.1528, "num_tokens": 1287202588.0, "reward": 0.5993303805589676, "reward_std": 0.14417463168501854, "rewards/accuracy_reward/mean": 0.10937500186264515, "rewards/accuracy_reward/std": 0.2985406033694744, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 2683 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375000000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 822.7522583007812, "completions/mean_terminated_length": 720.3460998535156, "completions/min_length": 287.25, "completions/min_terminated_length": 287.25, "epoch": 0.8017325069076245, "grad_norm": 0.45237138867378235, "kl": 3.12890625, "learning_rate": 1.5485476449124225e-06, "loss": 0.1654, "num_tokens": 1287650573.0, "reward": 0.5546875298023224, "reward_std": 0.12324998900294304, "rewards/accuracy_reward/mean": 0.06696428544819355, "rewards/accuracy_reward/std": 0.2395193688571453, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053606295958161354, "step": 2684 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 796.1272735595703, "completions/mean_terminated_length": 695.5401306152344, "completions/min_length": 269.75, "completions/min_terminated_length": 269.75, "epoch": 0.802031214995146, "grad_norm": 0.31433388590812683, "kl": 1.830078125, "learning_rate": 1.542721782960268e-06, "loss": 0.0972, "num_tokens": 1288071622.0, "reward": 0.710379496216774, "reward_std": 0.16759852319955826, "rewards/accuracy_reward/mean": 0.2165178582072258, "rewards/accuracy_reward/std": 0.3973629102110863, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037321708630770445, "step": 2685 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 832.4598541259766, "completions/mean_terminated_length": 730.8561248779297, "completions/min_length": 267.0, "completions/min_terminated_length": 267.0, "epoch": 0.8023299230826675, "grad_norm": 0.24271342158317566, "kl": 2.71875, "learning_rate": 1.5369059844222279e-06, "loss": 0.1446, "num_tokens": 1288514484.0, "reward": 0.632254496216774, "reward_std": 0.20825770124793053, "rewards/accuracy_reward/mean": 0.1450892835855484, "rewards/accuracy_reward/std": 0.343379657715559, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05396955460309982, "step": 2686 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2700892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 799.1004791259766, "completions/mean_terminated_length": 719.3493041992188, "completions/min_length": 353.25, "completions/min_terminated_length": 353.25, "epoch": 0.8026286311701889, "grad_norm": 0.2220194786787033, "kl": 2.404296875, "learning_rate": 1.5311002562185905e-06, "loss": 0.1317, "num_tokens": 1288955953.0, "reward": 0.766183078289032, "reward_std": 0.1913374923169613, "rewards/accuracy_reward/mean": 0.2745535746216774, "rewards/accuracy_reward/std": 0.43434546887874603, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 2687 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 816.2076416015625, "completions/mean_terminated_length": 706.0377502441406, "completions/min_length": 359.25, "completions/min_terminated_length": 359.25, "epoch": 0.8029273392577104, "grad_norm": 0.6296322345733643, "kl": 2.388671875, "learning_rate": 1.5253046052576559e-06, "loss": 0.1615, "num_tokens": 1289396990.0, "reward": 0.803013414144516, "reward_std": 0.19341618195176125, "rewards/accuracy_reward/mean": 0.3214285746216774, "rewards/accuracy_reward/std": 0.4567718356847763, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.050299433059990406, "step": 2688 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 845.0089569091797, "completions/mean_terminated_length": 731.8470916748047, "completions/min_length": 220.75, "completions/min_terminated_length": 220.75, "epoch": 0.8032260473452318, "grad_norm": 0.38064002990722656, "kl": 2.013671875, "learning_rate": 1.5195190384357405e-06, "loss": 0.1136, "num_tokens": 1289849010.0, "reward": 0.6266741305589676, "reward_std": 0.12352537736296654, "rewards/accuracy_reward/mean": 0.13616071385331452, "rewards/accuracy_reward/std": 0.30374230816960335, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04078465234488249, "step": 2689 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2834821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 806.294677734375, "completions/mean_terminated_length": 724.1343841552734, "completions/min_length": 379.25, "completions/min_terminated_length": 379.25, "epoch": 0.8035247554327534, "grad_norm": 0.31070420145988464, "kl": 1.568359375, "learning_rate": 1.5137435626371567e-06, "loss": 0.0953, "num_tokens": 1290286758.0, "reward": 0.7410714775323868, "reward_std": 0.2056597233749926, "rewards/accuracy_reward/mean": 0.2477678582072258, "rewards/accuracy_reward/std": 0.3451505973935127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2690 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 867.4888763427734, "completions/mean_terminated_length": 765.1286926269531, "completions/min_length": 226.0, "completions/min_terminated_length": 226.0, "epoch": 0.8038234635202748, "grad_norm": 0.2055366188287735, "kl": 1.93359375, "learning_rate": 1.5079781847342122e-06, "loss": 0.0869, "num_tokens": 1290758625.0, "reward": 0.729910746216774, "reward_std": 0.1520554912276566, "rewards/accuracy_reward/mean": 0.2388392873108387, "rewards/accuracy_reward/std": 0.3914143033325672, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 2691 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 804.8482666015625, "completions/mean_terminated_length": 692.50439453125, "completions/min_length": 266.5, "completions/min_terminated_length": 266.5, "epoch": 0.8041221716077963, "grad_norm": 0.33999356627464294, "kl": 1.60546875, "learning_rate": 1.5022229115871933e-06, "loss": 0.0994, "num_tokens": 1291185293.0, "reward": 0.675223246216774, "reward_std": 0.1525085810571909, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.31443919986486435, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 2692 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 801.8861999511719, "completions/mean_terminated_length": 688.1560974121094, "completions/min_length": 350.25, "completions/min_terminated_length": 350.25, "epoch": 0.8044208796953177, "grad_norm": 0.36283597350120544, "kl": 2.40234375, "learning_rate": 1.4964777500443728e-06, "loss": 0.1337, "num_tokens": 1291621514.0, "reward": 0.7154018133878708, "reward_std": 0.2055276334285736, "rewards/accuracy_reward/mean": 0.2254464291036129, "rewards/accuracy_reward/std": 0.4042694643139839, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 2693 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 866.0781707763672, "completions/mean_terminated_length": 773.0379333496094, "completions/min_length": 298.0, "completions/min_terminated_length": 298.0, "epoch": 0.8047195877828393, "grad_norm": 0.3020022511482239, "kl": 1.49462890625, "learning_rate": 1.4907427069419789e-06, "loss": 0.0853, "num_tokens": 1292089773.0, "reward": 0.6964286118745804, "reward_std": 0.16977556515485048, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.33092011511325836, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03475248999893665, "step": 2694 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 874.4866485595703, "completions/mean_terminated_length": 773.7567901611328, "completions/min_length": 409.25, "completions/min_terminated_length": 409.25, "epoch": 0.8050182958703607, "grad_norm": 0.3358312249183655, "kl": 1.56640625, "learning_rate": 1.4850177891042128e-06, "loss": 0.0907, "num_tokens": 1292562647.0, "reward": 0.690848246216774, "reward_std": 0.21131537109613419, "rewards/accuracy_reward/mean": 0.19866071455180645, "rewards/accuracy_reward/std": 0.39193740487098694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04357414972037077, "step": 2695 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42633928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 851.3839569091797, "completions/mean_terminated_length": 731.1680297851562, "completions/min_length": 219.0, "completions/min_terminated_length": 219.0, "epoch": 0.8053170039578822, "grad_norm": 0.8248617649078369, "kl": 3.02734375, "learning_rate": 1.4793030033432143e-06, "loss": 0.1637, "num_tokens": 1293020211.0, "reward": 0.5680803880095482, "reward_std": 0.09902025200426579, "rewards/accuracy_reward/mean": 0.08258928405120969, "rewards/accuracy_reward/std": 0.213464867323637, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.055894264951348305, "step": 2696 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 995.25, "completions/mean_length": 835.1406555175781, "completions/mean_terminated_length": 730.145751953125, "completions/min_length": 378.75, "completions/min_terminated_length": 378.75, "epoch": 0.8056157120454036, "grad_norm": 0.18278250098228455, "kl": 1.921875, "learning_rate": 1.4735983564590784e-06, "loss": 0.0934, "num_tokens": 1293459186.0, "reward": 0.6266741305589676, "reward_std": 0.13208580017089844, "rewards/accuracy_reward/mean": 0.13616071175783873, "rewards/accuracy_reward/std": 0.2784716933965683, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.044667139649391174, "step": 2697 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 901.8728179931641, "completions/mean_terminated_length": 814.1260223388672, "completions/min_length": 510.25, "completions/min_terminated_length": 510.25, "epoch": 0.8059144201329251, "grad_norm": 0.21251218020915985, "kl": 1.615234375, "learning_rate": 1.4679038552398294e-06, "loss": 0.0767, "num_tokens": 1293932361.0, "reward": 0.6121652126312256, "reward_std": 0.14664194220677018, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.26360035315155983, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 2698 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 806.3125305175781, "completions/mean_terminated_length": 711.7017364501953, "completions/min_length": 318.75, "completions/min_terminated_length": 318.75, "epoch": 0.8062131282204466, "grad_norm": 0.2517501711845398, "kl": 2.1005859375, "learning_rate": 1.4622195064614241e-06, "loss": 0.119, "num_tokens": 1294363317.0, "reward": 0.6685268133878708, "reward_std": 0.18477299809455872, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.3728727698326111, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.03758151177316904, "step": 2699 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 822.8839569091797, "completions/mean_terminated_length": 713.7477874755859, "completions/min_length": 298.5, "completions/min_terminated_length": 298.5, "epoch": 0.8065118363079681, "grad_norm": 0.42574048042297363, "kl": 2.2275390625, "learning_rate": 1.4565453168877297e-06, "loss": 0.1426, "num_tokens": 1294809521.0, "reward": 0.7008928805589676, "reward_std": 0.18071746081113815, "rewards/accuracy_reward/mean": 0.21205356810241938, "rewards/accuracy_reward/std": 0.36530132591724396, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982072740793, "step": 2700 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 989.5, "completions/mean_length": 828.7902069091797, "completions/mean_terminated_length": 713.8683624267578, "completions/min_length": 383.25, "completions/min_terminated_length": 383.25, "epoch": 0.8068105443954895, "grad_norm": 0.47242578864097595, "kl": 2.669921875, "learning_rate": 1.4508812932705364e-06, "loss": 0.1774, "num_tokens": 1295252339.0, "reward": 0.8270089626312256, "reward_std": 0.26747846230864525, "rewards/accuracy_reward/mean": 0.3392857164144516, "rewards/accuracy_reward/std": 0.4285849928855896, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.0424918862991035, "step": 2701 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 813.0178985595703, "completions/mean_terminated_length": 731.1624603271484, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.807109252483011, "grad_norm": 0.2511173188686371, "kl": 1.671875, "learning_rate": 1.445227442349526e-06, "loss": 0.0935, "num_tokens": 1295681835.0, "reward": 0.6406250298023224, "reward_std": 0.13812477886676788, "rewards/accuracy_reward/mean": 0.1536458320915699, "rewards/accuracy_reward/std": 0.3496636562049389, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03992978110909462, "step": 2702 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 872.8460388183594, "completions/mean_terminated_length": 763.1415100097656, "completions/min_length": 318.75, "completions/min_terminated_length": 318.75, "epoch": 0.8074079605705324, "grad_norm": 0.3839477598667145, "kl": 2.890625, "learning_rate": 1.4395837708522864e-06, "loss": 0.1346, "num_tokens": 1296147494.0, "reward": 0.6222098469734192, "reward_std": 0.1393412295728922, "rewards/accuracy_reward/mean": 0.1361607131548226, "rewards/accuracy_reward/std": 0.30715595558285713, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.055920460261404514, "step": 2703 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 858.2611999511719, "completions/mean_terminated_length": 743.1308898925781, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "epoch": 0.807706668658054, "grad_norm": 0.28446412086486816, "kl": 2.259765625, "learning_rate": 1.4339502854942866e-06, "loss": 0.1036, "num_tokens": 1296610667.0, "reward": 0.7237723618745804, "reward_std": 0.21737894415855408, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.42354054749011993, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 2704 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 855.7388763427734, "completions/mean_terminated_length": 728.3538513183594, "completions/min_length": 299.5, "completions/min_terminated_length": 299.5, "epoch": 0.8080053767455754, "grad_norm": 0.24384231865406036, "kl": 2.517578125, "learning_rate": 1.4283269929788779e-06, "loss": 0.1425, "num_tokens": 1297075494.0, "reward": 0.6222098618745804, "reward_std": 0.13288595341145992, "rewards/accuracy_reward/mean": 0.1316964258439839, "rewards/accuracy_reward/std": 0.3033906929194927, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04537529917433858, "step": 2705 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 827.7299499511719, "completions/mean_terminated_length": 705.1117858886719, "completions/min_length": 354.25, "completions/min_terminated_length": 354.25, "epoch": 0.8083040848330969, "grad_norm": 0.33466213941574097, "kl": 2.51953125, "learning_rate": 1.4227138999972801e-06, "loss": 0.1316, "num_tokens": 1297520701.0, "reward": 0.6668527126312256, "reward_std": 0.17707098880782723, "rewards/accuracy_reward/mean": 0.17633928917348385, "rewards/accuracy_reward/std": 0.3075162097811699, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 2706 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 874.6205749511719, "completions/mean_terminated_length": 756.6690673828125, "completions/min_length": 310.0, "completions/min_terminated_length": 310.0, "epoch": 0.8086027929206183, "grad_norm": 0.2656733989715576, "kl": 2.01171875, "learning_rate": 1.4171110132285771e-06, "loss": 0.1007, "num_tokens": 1297985859.0, "reward": 0.6428571790456772, "reward_std": 0.14784033223986626, "rewards/accuracy_reward/mean": 0.1517857159487903, "rewards/accuracy_reward/std": 0.29721618816256523, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589571990072727, "step": 2707 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 810.3125305175781, "completions/mean_terminated_length": 694.961181640625, "completions/min_length": 305.75, "completions/min_terminated_length": 305.75, "epoch": 0.8089015010081398, "grad_norm": 0.3800273835659027, "kl": 2.365234375, "learning_rate": 1.4115183393397147e-06, "loss": 0.1385, "num_tokens": 1298423759.0, "reward": 0.7209821790456772, "reward_std": 0.14339753426611423, "rewards/accuracy_reward/mean": 0.2321428540162742, "rewards/accuracy_reward/std": 0.3703354597091675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04800479579716921, "step": 2708 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 845.8326263427734, "completions/mean_terminated_length": 722.9927978515625, "completions/min_length": 356.75, "completions/min_terminated_length": 356.75, "epoch": 0.8092002090956613, "grad_norm": 0.2947484850883484, "kl": 2.1748046875, "learning_rate": 1.4059358849854732e-06, "loss": 0.1284, "num_tokens": 1298870948.0, "reward": 0.5775670111179352, "reward_std": 0.13714842684566975, "rewards/accuracy_reward/mean": 0.08705357136204839, "rewards/accuracy_reward/std": 0.2687148116528988, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04597289999946952, "step": 2709 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 871.5580596923828, "completions/mean_terminated_length": 752.2391815185547, "completions/min_length": 400.75, "completions/min_terminated_length": 400.75, "epoch": 0.8094989171831828, "grad_norm": 0.36527785658836365, "kl": 2.240234375, "learning_rate": 1.4003636568084877e-06, "loss": 0.1345, "num_tokens": 1299350046.0, "reward": 0.6462053805589676, "reward_std": 0.1714305728673935, "rewards/accuracy_reward/mean": 0.15401785564608872, "rewards/accuracy_reward/std": 0.31344777159392834, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.042172474320977926, "step": 2710 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 894.1719055175781, "completions/mean_terminated_length": 745.9782104492188, "completions/min_length": 267.5, "completions/min_terminated_length": 267.5, "epoch": 0.8097976252707042, "grad_norm": 0.22442485392093658, "kl": 2.20703125, "learning_rate": 1.3948016614392113e-06, "loss": 0.1079, "num_tokens": 1299824043.0, "reward": 0.5714285969734192, "reward_std": 0.13717170245945454, "rewards/accuracy_reward/mean": 0.08258928777649999, "rewards/accuracy_reward/std": 0.25489699095487595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051264057867228985, "step": 2711 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 866.1540679931641, "completions/mean_terminated_length": 741.9450225830078, "completions/min_length": 392.75, "completions/min_terminated_length": 392.75, "epoch": 0.8100963333582257, "grad_norm": 0.316155344247818, "kl": 2.072265625, "learning_rate": 1.3892499054959296e-06, "loss": 0.114, "num_tokens": 1300290608.0, "reward": 0.6255580633878708, "reward_std": 0.08799928426742554, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.33847761899232864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 2712 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 862.3370819091797, "completions/mean_terminated_length": 753.0836334228516, "completions/min_length": 380.0, "completions/min_terminated_length": 380.0, "epoch": 0.8103950414457471, "grad_norm": 0.28709715604782104, "kl": 2.564453125, "learning_rate": 1.3837083955847418e-06, "loss": 0.131, "num_tokens": 1300754071.0, "reward": 0.6356027126312256, "reward_std": 0.12448679655790329, "rewards/accuracy_reward/mean": 0.14732142770662904, "rewards/accuracy_reward/std": 0.31238868832588196, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 2713 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5290178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 891.5268249511719, "completions/mean_terminated_length": 743.5217590332031, "completions/min_length": 386.75, "completions/min_terminated_length": 386.75, "epoch": 0.8106937495332686, "grad_norm": 0.2515418827533722, "kl": 2.033203125, "learning_rate": 1.378177138299558e-06, "loss": 0.1057, "num_tokens": 1301224643.0, "reward": 0.6283482313156128, "reward_std": 0.13963391445577145, "rewards/accuracy_reward/mean": 0.13839285634458065, "rewards/accuracy_reward/std": 0.3411950170993805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 2714 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 866.9241485595703, "completions/mean_terminated_length": 757.0487670898438, "completions/min_length": 397.75, "completions/min_terminated_length": 397.75, "epoch": 0.8109924576207901, "grad_norm": 0.357987642288208, "kl": 2.5078125, "learning_rate": 1.3726561402220818e-06, "loss": 0.1384, "num_tokens": 1301684689.0, "reward": 0.6467634290456772, "reward_std": 0.20886485278606415, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.358648881316185, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05263457726687193, "step": 2715 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3504464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 826.4397735595703, "completions/mean_terminated_length": 723.2506561279297, "completions/min_length": 380.75, "completions/min_terminated_length": 380.75, "epoch": 0.8112911657083115, "grad_norm": 0.29034891724586487, "kl": 2.044921875, "learning_rate": 1.3671454079218171e-06, "loss": 0.1154, "num_tokens": 1302132326.0, "reward": 0.680245578289032, "reward_std": 0.1266610138118267, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.39207982271909714, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241046637297, "step": 2716 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47544642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 895.1116638183594, "completions/mean_terminated_length": 783.9610595703125, "completions/min_length": 365.75, "completions/min_terminated_length": 365.75, "epoch": 0.811589873795833, "grad_norm": 0.2310481071472168, "kl": 2.93359375, "learning_rate": 1.3616449479560434e-06, "loss": 0.1463, "num_tokens": 1302608552.0, "reward": 0.5446428805589676, "reward_std": 0.12401876971125603, "rewards/accuracy_reward/mean": 0.058035714784637094, "rewards/accuracy_reward/std": 0.2186268512159586, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05598148982971907, "step": 2717 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 850.9933319091797, "completions/mean_terminated_length": 720.1870880126953, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "epoch": 0.8118885818833544, "grad_norm": 0.2841939628124237, "kl": 2.13671875, "learning_rate": 1.356154766869826e-06, "loss": 0.1216, "num_tokens": 1303055301.0, "reward": 0.5797991305589676, "reward_std": 0.10667060501873493, "rewards/accuracy_reward/mean": 0.08928571734577417, "rewards/accuracy_reward/std": 0.2505672201514244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04672335181385279, "step": 2718 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 849.5603179931641, "completions/mean_terminated_length": 705.602294921875, "completions/min_length": 263.75, "completions/min_terminated_length": 263.75, "epoch": 0.812187289970876, "grad_norm": 0.2810419499874115, "kl": 2.154296875, "learning_rate": 1.350674871195995e-06, "loss": 0.095, "num_tokens": 1303508400.0, "reward": 0.7265625298023224, "reward_std": 0.1824091114103794, "rewards/accuracy_reward/mean": 0.23660714784637094, "rewards/accuracy_reward/std": 0.3740975074470043, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 2719 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 880.4576110839844, "completions/mean_terminated_length": 744.0888061523438, "completions/min_length": 341.5, "completions/min_terminated_length": 341.5, "epoch": 0.8124859980583974, "grad_norm": 0.25268852710723877, "kl": 1.912109375, "learning_rate": 1.345205267455143e-06, "loss": 0.0985, "num_tokens": 1303972445.0, "reward": 0.6579241305589676, "reward_std": 0.13826914364472032, "rewards/accuracy_reward/mean": 0.1796875, "rewards/accuracy_reward/std": 0.30555395036935806, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 2720 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 851.7946624755859, "completions/mean_terminated_length": 722.6584625244141, "completions/min_length": 335.75, "completions/min_terminated_length": 335.75, "epoch": 0.8127847061459189, "grad_norm": 0.2350151240825653, "kl": 2.546875, "learning_rate": 1.339745962155613e-06, "loss": 0.1379, "num_tokens": 1304426929.0, "reward": 0.595982164144516, "reward_std": 0.185462586581707, "rewards/accuracy_reward/mean": 0.10937499930150807, "rewards/accuracy_reward/std": 0.2863962110131979, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05596293695271015, "step": 2721 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 885.7344207763672, "completions/mean_terminated_length": 774.5255737304688, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "epoch": 0.8130834142334403, "grad_norm": 0.25372758507728577, "kl": 2.65625, "learning_rate": 1.3342969617934998e-06, "loss": 0.1473, "num_tokens": 1304899882.0, "reward": 0.6356027126312256, "reward_std": 0.17335942201316357, "rewards/accuracy_reward/mean": 0.1502976231276989, "rewards/accuracy_reward/std": 0.3412783369421959, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052634578198194504, "step": 2722 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 858.2388916015625, "completions/mean_terminated_length": 729.9720001220703, "completions/min_length": 383.0, "completions/min_terminated_length": 383.0, "epoch": 0.8133821223209619, "grad_norm": 0.37463369965553284, "kl": 2.52734375, "learning_rate": 1.3288582728526277e-06, "loss": 0.1268, "num_tokens": 1305352581.0, "reward": 0.7287946790456772, "reward_std": 0.12354222126305103, "rewards/accuracy_reward/mean": 0.2388392835855484, "rewards/accuracy_reward/std": 0.4131934717297554, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048634594306349754, "step": 2723 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 827.0357513427734, "completions/mean_terminated_length": 722.5793304443359, "completions/min_length": 296.25, "completions/min_terminated_length": 296.25, "epoch": 0.8136808304084833, "grad_norm": 0.342843621969223, "kl": 2.69140625, "learning_rate": 1.3234299018045615e-06, "loss": 0.1628, "num_tokens": 1305793909.0, "reward": 0.7126116305589676, "reward_std": 0.17149090208113194, "rewards/accuracy_reward/mean": 0.2254464253783226, "rewards/accuracy_reward/std": 0.4103968143463135, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05252148862928152, "step": 2724 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 878.9107513427734, "completions/mean_terminated_length": 752.8121490478516, "completions/min_length": 325.25, "completions/min_terminated_length": 325.25, "epoch": 0.8139795384960048, "grad_norm": 0.7036795616149902, "kl": 2.544921875, "learning_rate": 1.3180118551085763e-06, "loss": 0.1157, "num_tokens": 1306264653.0, "reward": 0.6166294813156128, "reward_std": 0.15684566646814346, "rewards/accuracy_reward/mean": 0.12723214458674192, "rewards/accuracy_reward/std": 0.31238991022109985, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.049098861403763294, "step": 2725 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5357142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 897.982177734375, "completions/mean_terminated_length": 740.8240814208984, "completions/min_length": 273.75, "completions/min_terminated_length": 273.75, "epoch": 0.8142782465835262, "grad_norm": 0.2409103363752365, "kl": 2.28125, "learning_rate": 1.3126041392116774e-06, "loss": 0.1142, "num_tokens": 1306749749.0, "reward": 0.6629464477300644, "reward_std": 0.09102505445480347, "rewards/accuracy_reward/mean": 0.17187499743886292, "rewards/accuracy_reward/std": 0.3367245849221945, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 2726 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 918.8370819091797, "completions/mean_terminated_length": 807.1510009765625, "completions/min_length": 441.25, "completions/min_terminated_length": 441.25, "epoch": 0.8145769546710477, "grad_norm": 0.25335022807121277, "kl": 2.02734375, "learning_rate": 1.3072067605485628e-06, "loss": 0.0929, "num_tokens": 1307234764.0, "reward": 0.5680803880095482, "reward_std": 0.11022728052921593, "rewards/accuracy_reward/mean": 0.07812499813735485, "rewards/accuracy_reward/std": 0.1802080199122429, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04558529471978545, "step": 2727 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 844.450927734375, "completions/mean_terminated_length": 732.9739532470703, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "epoch": 0.8148756627585692, "grad_norm": 0.28230369091033936, "kl": 1.4697265625, "learning_rate": 1.301819725541641e-06, "loss": 0.0863, "num_tokens": 1307688614.0, "reward": 0.6489955633878708, "reward_std": 0.13044499419629574, "rewards/accuracy_reward/mean": 0.15401785308495164, "rewards/accuracy_reward/std": 0.2625789977610111, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.02461609710007906, "step": 2728 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 848.4219055175781, "completions/mean_terminated_length": 722.3997039794922, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "epoch": 0.8151743708460907, "grad_norm": 0.3502584397792816, "kl": 2.373046875, "learning_rate": 1.2964430406010032e-06, "loss": 0.1371, "num_tokens": 1308150819.0, "reward": 0.6607143133878708, "reward_std": 0.168094951659441, "rewards/accuracy_reward/mean": 0.1696428577415645, "rewards/accuracy_reward/std": 0.3399353474378586, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04355311533436179, "step": 2729 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 822.3906555175781, "completions/mean_terminated_length": 701.3050994873047, "completions/min_length": 280.75, "completions/min_terminated_length": 280.75, "epoch": 0.8154730789336121, "grad_norm": 0.37454357743263245, "kl": 2.197265625, "learning_rate": 1.2910767121244349e-06, "loss": 0.1059, "num_tokens": 1308590546.0, "reward": 0.6702009290456772, "reward_std": 0.1845523566007614, "rewards/accuracy_reward/mean": 0.1785714253783226, "rewards/accuracy_reward/std": 0.3565348722040653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294738650322, "rewards/tag_count_reward/std": 0.04348720656707883, "step": 2730 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 891.6830749511719, "completions/mean_terminated_length": 776.2494506835938, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.8157717870211336, "grad_norm": 0.3373175263404846, "kl": 2.1142578125, "learning_rate": 1.2857207464973876e-06, "loss": 0.1084, "num_tokens": 1309060436.0, "reward": 0.5569196715950966, "reward_std": 0.1236461317166686, "rewards/accuracy_reward/mean": 0.06696428637951612, "rewards/accuracy_reward/std": 0.19648104161024094, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 2731 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 861.5580749511719, "completions/mean_terminated_length": 745.3469390869141, "completions/min_length": 343.5, "completions/min_terminated_length": 343.5, "epoch": 0.816070495108655, "grad_norm": 0.2988870441913605, "kl": 2.224609375, "learning_rate": 1.2803751500929895e-06, "loss": 0.1165, "num_tokens": 1309526254.0, "reward": 0.742745578289032, "reward_std": 0.20178688317537308, "rewards/accuracy_reward/mean": 0.25223214365541935, "rewards/accuracy_reward/std": 0.40986741334199905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04534579161554575, "step": 2732 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 863.1786193847656, "completions/mean_terminated_length": 723.5817108154297, "completions/min_length": 326.75, "completions/min_terminated_length": 326.75, "epoch": 0.8163692031961766, "grad_norm": 0.41766607761383057, "kl": 2.5703125, "learning_rate": 1.2750399292720284e-06, "loss": 0.1378, "num_tokens": 1309983438.0, "reward": 0.6328125447034836, "reward_std": 0.12347390688955784, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.33305684104561806, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.05409308057278395, "step": 2733 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 810.3772735595703, "completions/mean_terminated_length": 708.8263854980469, "completions/min_length": 326.25, "completions/min_terminated_length": 326.25, "epoch": 0.816667911283698, "grad_norm": 0.3128666877746582, "kl": 2.462890625, "learning_rate": 1.269715090382948e-06, "loss": 0.1149, "num_tokens": 1310416743.0, "reward": 0.7159598618745804, "reward_std": 0.19491872563958168, "rewards/accuracy_reward/mean": 0.2276785671710968, "rewards/accuracy_reward/std": 0.4067734330892563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.054497321136295795, "step": 2734 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45982142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 880.7299346923828, "completions/mean_terminated_length": 764.1564788818359, "completions/min_length": 434.0, "completions/min_terminated_length": 434.0, "epoch": 0.8169666193712195, "grad_norm": 0.4260748326778412, "kl": 1.3271484375, "learning_rate": 1.2644006397618325e-06, "loss": 0.0666, "num_tokens": 1310880046.0, "reward": 0.7031250298023224, "reward_std": 0.19524306245148182, "rewards/accuracy_reward/mean": 0.217261902987957, "rewards/accuracy_reward/std": 0.4083936735987663, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.031776280142366886, "step": 2735 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 858.2567291259766, "completions/mean_terminated_length": 767.1186828613281, "completions/min_length": 490.25, "completions/min_terminated_length": 490.25, "epoch": 0.8172653274587409, "grad_norm": 0.23483552038669586, "kl": 1.9208984375, "learning_rate": 1.2590965837324132e-06, "loss": 0.0866, "num_tokens": 1311334081.0, "reward": 0.6925223469734192, "reward_std": 0.16876980662345886, "rewards/accuracy_reward/mean": 0.20089285681024194, "rewards/accuracy_reward/std": 0.3564968965947628, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.042492654640227556, "step": 2736 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 818.2187805175781, "completions/mean_terminated_length": 711.0065155029297, "completions/min_length": 337.75, "completions/min_terminated_length": 337.75, "epoch": 0.8175640355462624, "grad_norm": 0.3110978305339813, "kl": 2.310546875, "learning_rate": 1.2538029286060428e-06, "loss": 0.1259, "num_tokens": 1311780243.0, "reward": 0.6132812649011612, "reward_std": 0.14162501879036427, "rewards/accuracy_reward/mean": 0.1227678582072258, "rewards/accuracy_reward/std": 0.3252161666750908, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 2737 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 830.1986999511719, "completions/mean_terminated_length": 722.4862518310547, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 0.8178627436337839, "grad_norm": 0.3646399676799774, "kl": 1.837890625, "learning_rate": 1.248519680681709e-06, "loss": 0.109, "num_tokens": 1312224060.0, "reward": 0.6841518133878708, "reward_std": 0.17579646036028862, "rewards/accuracy_reward/mean": 0.19196428451687098, "rewards/accuracy_reward/std": 0.3639809414744377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04337459057569504, "step": 2738 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 898.3638763427734, "completions/mean_terminated_length": 766.1486358642578, "completions/min_length": 442.75, "completions/min_terminated_length": 442.75, "epoch": 0.8181614517213054, "grad_norm": 0.22546763718128204, "kl": 1.775390625, "learning_rate": 1.2432468462460024e-06, "loss": 0.0895, "num_tokens": 1312697951.0, "reward": 0.599888414144516, "reward_std": 0.11303550843149424, "rewards/accuracy_reward/mean": 0.10714285750873387, "rewards/accuracy_reward/std": 0.27403895929455757, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196681171656, "step": 2739 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 899.0156555175781, "completions/mean_terminated_length": 775.2169036865234, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "epoch": 0.8184601598088268, "grad_norm": 0.3153793513774872, "kl": 2.5078125, "learning_rate": 1.2379844315731393e-06, "loss": 0.1189, "num_tokens": 1313177654.0, "reward": 0.6847098469734192, "reward_std": 0.2182098962366581, "rewards/accuracy_reward/mean": 0.19642857369035482, "rewards/accuracy_reward/std": 0.355152003467083, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052634578198194504, "step": 2740 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 833.044677734375, "completions/mean_terminated_length": 723.1169891357422, "completions/min_length": 357.75, "completions/min_terminated_length": 357.75, "epoch": 0.8187588678963483, "grad_norm": 0.3276032507419586, "kl": 2.859375, "learning_rate": 1.2327324429249232e-06, "loss": 0.1609, "num_tokens": 1313624730.0, "reward": 0.5876116305589676, "reward_std": 0.157800804823637, "rewards/accuracy_reward/mean": 0.10044642817229033, "rewards/accuracy_reward/std": 0.29123885184526443, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 2741 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 845.5714721679688, "completions/mean_terminated_length": 728.7523651123047, "completions/min_length": 325.5, "completions/min_terminated_length": 325.5, "epoch": 0.8190575759838697, "grad_norm": 0.26100361347198486, "kl": 3.07421875, "learning_rate": 1.2274908865507595e-06, "loss": 0.1523, "num_tokens": 1314070986.0, "reward": 0.694754496216774, "reward_std": 0.14976156130433083, "rewards/accuracy_reward/mean": 0.20982142654247582, "rewards/accuracy_reward/std": 0.35600471310317516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05856847669929266, "step": 2742 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 870.2701263427734, "completions/mean_terminated_length": 759.4453125, "completions/min_length": 345.75, "completions/min_terminated_length": 345.75, "epoch": 0.8193562840713913, "grad_norm": 0.28104138374328613, "kl": 2.19140625, "learning_rate": 1.2222597686876337e-06, "loss": 0.106, "num_tokens": 1314536371.0, "reward": 0.6305803656578064, "reward_std": 0.1726619377732277, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.32112155109643936, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04669034807011485, "step": 2743 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 874.3616638183594, "completions/mean_terminated_length": 769.2979125976562, "completions/min_length": 439.75, "completions/min_terminated_length": 439.75, "epoch": 0.8196549921589127, "grad_norm": 0.45866984128952026, "kl": 1.6396484375, "learning_rate": 1.2170390955601175e-06, "loss": 0.0892, "num_tokens": 1314996725.0, "reward": 0.643973246216774, "reward_std": 0.14372291415929794, "rewards/accuracy_reward/mean": 0.1517857094295323, "rewards/accuracy_reward/std": 0.3181196339428425, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.042415475472807884, "step": 2744 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 855.8705749511719, "completions/mean_terminated_length": 759.5126342773438, "completions/min_length": 313.25, "completions/min_terminated_length": 313.25, "epoch": 0.8199537002464342, "grad_norm": 0.21914882957935333, "kl": 1.2685546875, "learning_rate": 1.2118288733803474e-06, "loss": 0.067, "num_tokens": 1315449883.0, "reward": 0.781808078289032, "reward_std": 0.23851626366376877, "rewards/accuracy_reward/mean": 0.2879464253783226, "rewards/accuracy_reward/std": 0.41443702578544617, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.037829161155968904, "step": 2745 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 836.122802734375, "completions/mean_terminated_length": 752.5593566894531, "completions/min_length": 426.75, "completions/min_terminated_length": 426.75, "epoch": 0.8202524083339556, "grad_norm": 0.30920806527137756, "kl": 1.6533203125, "learning_rate": 1.2066291083480297e-06, "loss": 0.0731, "num_tokens": 1315894882.0, "reward": 0.6618304029107094, "reward_std": 0.16756187099963427, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.28811274468898773, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2746 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 869.4040374755859, "completions/mean_terminated_length": 757.5483703613281, "completions/min_length": 431.25, "completions/min_terminated_length": 431.25, "epoch": 0.8205511164214772, "grad_norm": 0.28821927309036255, "kl": 1.720703125, "learning_rate": 1.2014398066504263e-06, "loss": 0.0926, "num_tokens": 1316349799.0, "reward": 0.6668526977300644, "reward_std": 0.16727925091981888, "rewards/accuracy_reward/mean": 0.1741071459837258, "rewards/accuracy_reward/std": 0.32496409490704536, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 2747 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 824.9687805175781, "completions/mean_terminated_length": 724.9684143066406, "completions/min_length": 253.5, "completions/min_terminated_length": 253.5, "epoch": 0.8208498245089986, "grad_norm": 0.23277460038661957, "kl": 1.6640625, "learning_rate": 1.1962609744623476e-06, "loss": 0.0823, "num_tokens": 1316789225.0, "reward": 0.680245578289032, "reward_std": 0.18485024012625217, "rewards/accuracy_reward/mean": 0.18749999860301614, "rewards/accuracy_reward/std": 0.3522031642496586, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196681171656, "step": 2748 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 825.0647583007812, "completions/mean_terminated_length": 723.3013458251953, "completions/min_length": 284.5, "completions/min_terminated_length": 284.5, "epoch": 0.8211485325965201, "grad_norm": 0.21391130983829498, "kl": 1.100830078125, "learning_rate": 1.1910926179461446e-06, "loss": 0.0549, "num_tokens": 1317241702.0, "reward": 0.695870578289032, "reward_std": 0.17346962168812752, "rewards/accuracy_reward/mean": 0.20089286006987095, "rewards/accuracy_reward/std": 0.3949427753686905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.029593830928206444, "step": 2749 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.75, "completions/mean_length": 831.0714569091797, "completions/mean_terminated_length": 720.7516479492188, "completions/min_length": 280.25, "completions/min_terminated_length": 280.25, "epoch": 0.8214472406840415, "grad_norm": 0.43148738145828247, "kl": 2.083984375, "learning_rate": 1.1859347432517088e-06, "loss": 0.1189, "num_tokens": 1317680806.0, "reward": 0.6863839626312256, "reward_std": 0.14483616687357426, "rewards/accuracy_reward/mean": 0.19642856903374195, "rewards/accuracy_reward/std": 0.3610711097717285, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.049088423140347004, "step": 2750 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 826.779052734375, "completions/mean_terminated_length": 741.9583129882812, "completions/min_length": 269.75, "completions/min_terminated_length": 269.75, "epoch": 0.821745948771563, "grad_norm": 0.24770314991474152, "kl": 1.501953125, "learning_rate": 1.1807873565164507e-06, "loss": 0.0907, "num_tokens": 1318119539.0, "reward": 0.6852678954601288, "reward_std": 0.15878872387111187, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.39304138720035553, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03992978110909462, "step": 2751 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 818.7924499511719, "completions/mean_terminated_length": 713.8879241943359, "completions/min_length": 377.25, "completions/min_terminated_length": 377.25, "epoch": 0.8220446568590845, "grad_norm": 0.42898982763290405, "kl": 2.34765625, "learning_rate": 1.1756504638653087e-06, "loss": 0.1423, "num_tokens": 1318551238.0, "reward": 0.5898437798023224, "reward_std": 0.14595934748649597, "rewards/accuracy_reward/mean": 0.10267857275903225, "rewards/accuracy_reward/std": 0.24729007482528687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.056212532334029675, "step": 2752 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 860.1808319091797, "completions/mean_terminated_length": 751.2001800537109, "completions/min_length": 406.25, "completions/min_terminated_length": 406.25, "epoch": 0.822343364946606, "grad_norm": 0.2945561408996582, "kl": 2.10546875, "learning_rate": 1.1705240714107301e-06, "loss": 0.1215, "num_tokens": 1319007447.0, "reward": 0.6612723469734192, "reward_std": 0.18526408076286316, "rewards/accuracy_reward/mean": 0.1718750004656613, "rewards/accuracy_reward/std": 0.3249289430677891, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050148884765803814, "step": 2753 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 828.7031707763672, "completions/mean_terminated_length": 727.3573608398438, "completions/min_length": 356.0, "completions/min_terminated_length": 356.0, "epoch": 0.8226420730341274, "grad_norm": 0.32439985871315, "kl": 2.28125, "learning_rate": 1.165408185252671e-06, "loss": 0.1372, "num_tokens": 1319446594.0, "reward": 0.6607143133878708, "reward_std": 0.16842757537961006, "rewards/accuracy_reward/mean": 0.17187499720603228, "rewards/accuracy_reward/std": 0.34907444566488266, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050504449754953384, "step": 2754 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 840.9018249511719, "completions/mean_terminated_length": 716.6262969970703, "completions/min_length": 347.5, "completions/min_terminated_length": 347.5, "epoch": 0.8229407811216489, "grad_norm": 0.34053483605384827, "kl": 2.087890625, "learning_rate": 1.160302811478584e-06, "loss": 0.0966, "num_tokens": 1319897110.0, "reward": 0.8058036118745804, "reward_std": 0.18298587761819363, "rewards/accuracy_reward/mean": 0.3147321455180645, "rewards/accuracy_reward/std": 0.4616248086094856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04479066748172045, "step": 2755 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 830.107177734375, "completions/mean_terminated_length": 717.2206878662109, "completions/min_length": 300.0, "completions/min_terminated_length": 300.0, "epoch": 0.8232394892091703, "grad_norm": 0.2937672436237335, "kl": 2.951171875, "learning_rate": 1.1552079561634111e-06, "loss": 0.1809, "num_tokens": 1320336230.0, "reward": 0.7477678954601288, "reward_std": 0.30194445326924324, "rewards/accuracy_reward/mean": 0.26116071082651615, "rewards/accuracy_reward/std": 0.4152158126235008, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 2756 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34598214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 841.0156555175781, "completions/mean_terminated_length": 744.5662384033203, "completions/min_length": 308.0, "completions/min_terminated_length": 308.0, "epoch": 0.8235381972966918, "grad_norm": 0.24308153986930847, "kl": 2.494140625, "learning_rate": 1.1501236253695823e-06, "loss": 0.1349, "num_tokens": 1320786333.0, "reward": 0.7036830633878708, "reward_std": 0.21814491972327232, "rewards/accuracy_reward/mean": 0.21428571455180645, "rewards/accuracy_reward/std": 0.3967796191573143, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04909886047244072, "step": 2757 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 818.0625457763672, "completions/mean_terminated_length": 728.75146484375, "completions/min_length": 317.5, "completions/min_terminated_length": 317.5, "epoch": 0.8238369053842133, "grad_norm": 0.38023966550827026, "kl": 2.23046875, "learning_rate": 1.1450498251469988e-06, "loss": 0.1377, "num_tokens": 1321226025.0, "reward": 0.6573661118745804, "reward_std": 0.18565843999385834, "rewards/accuracy_reward/mean": 0.1674107159487903, "rewards/accuracy_reward/std": 0.3458382710814476, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.049088423140347004, "step": 2758 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 886.2254943847656, "completions/mean_terminated_length": 759.0313873291016, "completions/min_length": 439.25, "completions/min_terminated_length": 439.25, "epoch": 0.8241356134717347, "grad_norm": 0.30925998091697693, "kl": 1.6220703125, "learning_rate": 1.1399865615330397e-06, "loss": 0.0806, "num_tokens": 1321700334.0, "reward": 0.6908482611179352, "reward_std": 0.20129777491092682, "rewards/accuracy_reward/mean": 0.19866071455180645, "rewards/accuracy_reward/std": 0.38329601287841797, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04272336792200804, "step": 2759 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 850.4263916015625, "completions/mean_terminated_length": 738.4514770507812, "completions/min_length": 294.75, "completions/min_terminated_length": 294.75, "epoch": 0.8244343215592562, "grad_norm": 0.2839340269565582, "kl": 3.0859375, "learning_rate": 1.1349338405525368e-06, "loss": 0.1515, "num_tokens": 1322154077.0, "reward": 0.640066996216774, "reward_std": 0.1814489383250475, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.354690819978714, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05724119208753109, "step": 2760 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42857142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 884.3348693847656, "completions/mean_terminated_length": 782.2015533447266, "completions/min_length": 430.5, "completions/min_terminated_length": 430.5, "epoch": 0.8247330296467776, "grad_norm": 0.36857330799102783, "kl": 2.10546875, "learning_rate": 1.129891668217783e-06, "loss": 0.1117, "num_tokens": 1322623843.0, "reward": 0.6406250298023224, "reward_std": 0.14660402201116085, "rewards/accuracy_reward/mean": 0.14955356996506453, "rewards/accuracy_reward/std": 0.32551979273557663, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04560601245611906, "step": 2761 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47098214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 868.1808624267578, "completions/mean_terminated_length": 733.9185791015625, "completions/min_length": 345.5, "completions/min_terminated_length": 345.5, "epoch": 0.8250317377342992, "grad_norm": 0.29728275537490845, "kl": 1.1650390625, "learning_rate": 1.124860050528519e-06, "loss": 0.0654, "num_tokens": 1323085620.0, "reward": 0.616629496216774, "reward_std": 0.09245220245793462, "rewards/accuracy_reward/mean": 0.12276785634458065, "rewards/accuracy_reward/std": 0.2683769538998604, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.04022751050069928, "step": 2762 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 868.6875457763672, "completions/mean_terminated_length": 770.590576171875, "completions/min_length": 292.75, "completions/min_terminated_length": 292.75, "epoch": 0.8253304458218206, "grad_norm": 0.18753057718276978, "kl": 1.82421875, "learning_rate": 1.1198389934719277e-06, "loss": 0.0914, "num_tokens": 1323544488.0, "reward": 0.761160746216774, "reward_std": 0.2352719772607088, "rewards/accuracy_reward/mean": 0.270089291036129, "rewards/accuracy_reward/std": 0.43309851735830307, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 2763 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 870.8728179931641, "completions/mean_terminated_length": 754.6019592285156, "completions/min_length": 411.5, "completions/min_terminated_length": 411.5, "epoch": 0.8256291539093421, "grad_norm": 0.40491533279418945, "kl": 2.5634765625, "learning_rate": 1.114828503022618e-06, "loss": 0.1432, "num_tokens": 1324007455.0, "reward": 0.6947545111179352, "reward_std": 0.14894231595098972, "rewards/accuracy_reward/mean": 0.21428571757860482, "rewards/accuracy_reward/std": 0.3567338716238737, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.052379181142896414, "step": 2764 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35044642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 861.2924652099609, "completions/mean_terminated_length": 776.6646575927734, "completions/min_length": 405.75, "completions/min_terminated_length": 405.75, "epoch": 0.8259278619968635, "grad_norm": 0.22686892747879028, "kl": 1.78125, "learning_rate": 1.1098285851426372e-06, "loss": 0.0881, "num_tokens": 1324466354.0, "reward": 0.5792411118745804, "reward_std": 0.12307058461010456, "rewards/accuracy_reward/mean": 0.09002976445481181, "rewards/accuracy_reward/std": 0.2607058770954609, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.041961644776165485, "step": 2765 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2723214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 791.0201110839844, "completions/mean_terminated_length": 701.9096069335938, "completions/min_length": 191.25, "completions/min_terminated_length": 191.25, "epoch": 0.826226570084385, "grad_norm": 0.3248704969882965, "kl": 2.267578125, "learning_rate": 1.1048392457814406e-06, "loss": 0.1163, "num_tokens": 1324895339.0, "reward": 0.6914062947034836, "reward_std": 0.1658994797617197, "rewards/accuracy_reward/mean": 0.2008928582072258, "rewards/accuracy_reward/std": 0.39600296318531036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 2766 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5022321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 913.685302734375, "completions/mean_terminated_length": 810.3148956298828, "completions/min_length": 420.75, "completions/min_terminated_length": 420.75, "epoch": 0.8265252781719065, "grad_norm": 0.39778855443000793, "kl": 2.501953125, "learning_rate": 1.0998604908759025e-06, "loss": 0.1243, "num_tokens": 1325373998.0, "reward": 0.6233259290456772, "reward_std": 0.16992740146815777, "rewards/accuracy_reward/mean": 0.1361607164144516, "rewards/accuracy_reward/std": 0.3079971671104431, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 2767 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5535714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 910.6161193847656, "completions/mean_terminated_length": 777.2838745117188, "completions/min_length": 301.5, "completions/min_terminated_length": 301.5, "epoch": 0.826823986259428, "grad_norm": 0.29746323823928833, "kl": 1.427734375, "learning_rate": 1.0948923263503042e-06, "loss": 0.0801, "num_tokens": 1325860578.0, "reward": 0.607700914144516, "reward_std": 0.13239073753356934, "rewards/accuracy_reward/mean": 0.11383928451687098, "rewards/accuracy_reward/std": 0.29856502264738083, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 2768 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 873.8616485595703, "completions/mean_terminated_length": 773.6669006347656, "completions/min_length": 435.0, "completions/min_terminated_length": 435.0, "epoch": 0.8271226943469494, "grad_norm": 0.23956134915351868, "kl": 2.625, "learning_rate": 1.0899347581163222e-06, "loss": 0.1292, "num_tokens": 1326325492.0, "reward": 0.6378348469734192, "reward_std": 0.09826758923009038, "rewards/accuracy_reward/mean": 0.14955357275903225, "rewards/accuracy_reward/std": 0.34908226132392883, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05133028235286474, "step": 2769 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 862.9263916015625, "completions/mean_terminated_length": 755.6703948974609, "completions/min_length": 414.5, "completions/min_terminated_length": 414.5, "epoch": 0.8274214024344709, "grad_norm": 0.4710492789745331, "kl": 3.0859375, "learning_rate": 1.0849877920730212e-06, "loss": 0.1703, "num_tokens": 1326782995.0, "reward": 0.6579241454601288, "reward_std": 0.2180226892232895, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.37228500843048096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.05695320852100849, "step": 2770 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 880.9085083007812, "completions/mean_terminated_length": 765.2248077392578, "completions/min_length": 363.5, "completions/min_terminated_length": 363.5, "epoch": 0.8277201105219923, "grad_norm": 0.2641771137714386, "kl": 2.810546875, "learning_rate": 1.0800514341068592e-06, "loss": 0.1425, "num_tokens": 1327252090.0, "reward": 0.5920759290456772, "reward_std": 0.13320282846689224, "rewards/accuracy_reward/mean": 0.10491071199066937, "rewards/accuracy_reward/std": 0.2813630308955908, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.051525398157536983, "step": 2771 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 825.0469055175781, "completions/mean_terminated_length": 707.6744384765625, "completions/min_length": 265.25, "completions/min_terminated_length": 265.25, "epoch": 0.8280188186095139, "grad_norm": 0.38845375180244446, "kl": 1.845947265625, "learning_rate": 1.0751256900916607e-06, "loss": 0.0966, "num_tokens": 1327693823.0, "reward": 0.7661830633878708, "reward_std": 0.09289677161723375, "rewards/accuracy_reward/mean": 0.2745535708963871, "rewards/accuracy_reward/std": 0.43632128089666367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.03703158348798752, "step": 2772 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 872.0781555175781, "completions/mean_terminated_length": 757.9497528076172, "completions/min_length": 378.75, "completions/min_terminated_length": 378.75, "epoch": 0.8283175266970353, "grad_norm": 0.29653218388557434, "kl": 2.287109375, "learning_rate": 1.0702105658886318e-06, "loss": 0.1192, "num_tokens": 1328160514.0, "reward": 0.7059152126312256, "reward_std": 0.20233134552836418, "rewards/accuracy_reward/mean": 0.21428571175783873, "rewards/accuracy_reward/std": 0.3839149847626686, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 2773 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39732142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 850.1696624755859, "completions/mean_terminated_length": 736.2268676757812, "completions/min_length": 311.5, "completions/min_terminated_length": 311.5, "epoch": 0.8286162347845568, "grad_norm": 0.2758018374443054, "kl": 2.87109375, "learning_rate": 1.06530606734633e-06, "loss": 0.1439, "num_tokens": 1328613182.0, "reward": 0.6891741529107094, "reward_std": 0.16276069823652506, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.3308144509792328, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.057241193018853664, "step": 2774 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 889.2299499511719, "completions/mean_terminated_length": 770.1838836669922, "completions/min_length": 373.5, "completions/min_terminated_length": 373.5, "epoch": 0.8289149428720782, "grad_norm": 0.2777695953845978, "kl": 2.234375, "learning_rate": 1.060412200300679e-06, "loss": 0.1268, "num_tokens": 1329080469.0, "reward": 0.8136161118745804, "reward_std": 0.22873609513044357, "rewards/accuracy_reward/mean": 0.3236607164144516, "rewards/accuracy_reward/std": 0.4602160006761551, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2775 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 836.5000457763672, "completions/mean_terminated_length": 708.2646026611328, "completions/min_length": 339.5, "completions/min_terminated_length": 339.5, "epoch": 0.8292136509595998, "grad_norm": 0.22077278792858124, "kl": 2.5390625, "learning_rate": 1.0555289705749483e-06, "loss": 0.1311, "num_tokens": 1329527461.0, "reward": 0.7165178954601288, "reward_std": 0.12981104478240013, "rewards/accuracy_reward/mean": 0.22767856949940324, "rewards/accuracy_reward/std": 0.36790255084633827, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 2776 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 868.0803985595703, "completions/mean_terminated_length": 763.0470428466797, "completions/min_length": 319.25, "completions/min_terminated_length": 319.25, "epoch": 0.8295123590471212, "grad_norm": 0.3665066063404083, "kl": 2.671875, "learning_rate": 1.0506563839797501e-06, "loss": 0.1383, "num_tokens": 1329987001.0, "reward": 0.7399553954601288, "reward_std": 0.20966594573110342, "rewards/accuracy_reward/mean": 0.25223214365541935, "rewards/accuracy_reward/std": 0.4103560075163841, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.052645014598965645, "step": 2777 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33482142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 820.7254943847656, "completions/mean_terminated_length": 720.2275238037109, "completions/min_length": 397.75, "completions/min_terminated_length": 397.75, "epoch": 0.8298110671346427, "grad_norm": 0.28058379888534546, "kl": 3.333984375, "learning_rate": 1.045794446313031e-06, "loss": 0.18, "num_tokens": 1330429342.0, "reward": 0.749441996216774, "reward_std": 0.1775351483374834, "rewards/accuracy_reward/mean": 0.2633928582072258, "rewards/accuracy_reward/std": 0.42594850063323975, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.056698620319366455, "step": 2778 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 866.2053985595703, "completions/mean_terminated_length": 730.7136383056641, "completions/min_length": 402.75, "completions/min_terminated_length": 402.75, "epoch": 0.8301097752221641, "grad_norm": 0.5782434940338135, "kl": 3.841796875, "learning_rate": 1.0409431633600687e-06, "loss": 0.1786, "num_tokens": 1330890698.0, "reward": 0.5920759215950966, "reward_std": 0.1556005012243986, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.26314349472522736, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06058031413704157, "step": 2779 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 836.3571929931641, "completions/mean_terminated_length": 713.8271179199219, "completions/min_length": 278.75, "completions/min_terminated_length": 278.75, "epoch": 0.8304084833096856, "grad_norm": 0.4538406729698181, "kl": 2.2109375, "learning_rate": 1.0361025408934588e-06, "loss": 0.1269, "num_tokens": 1331332650.0, "reward": 0.667410746216774, "reward_std": 0.18389106169342995, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.36123713478446007, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04386132536455989, "step": 2780 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 875.4754943847656, "completions/mean_terminated_length": 765.0208435058594, "completions/min_length": 356.25, "completions/min_terminated_length": 356.25, "epoch": 0.8307071913972071, "grad_norm": 0.30094414949417114, "kl": 2.6953125, "learning_rate": 1.0312725846731174e-06, "loss": 0.1495, "num_tokens": 1331803935.0, "reward": 0.6160714477300644, "reward_std": 0.16001809388399124, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.3253997005522251, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05035856366157532, "step": 2781 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 887.013427734375, "completions/mean_terminated_length": 773.2143096923828, "completions/min_length": 469.5, "completions/min_terminated_length": 469.5, "epoch": 0.8310058994847286, "grad_norm": 0.3140491545200348, "kl": 2.060546875, "learning_rate": 1.026453300446264e-06, "loss": 0.1015, "num_tokens": 1332273445.0, "reward": 0.6339285969734192, "reward_std": 0.1258649481460452, "rewards/accuracy_reward/mean": 0.14285714295692742, "rewards/accuracy_reward/std": 0.31583631597459316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04479066655039787, "step": 2782 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 846.8437652587891, "completions/mean_terminated_length": 723.3764190673828, "completions/min_length": 348.0, "completions/min_terminated_length": 348.0, "epoch": 0.83130460757225, "grad_norm": 0.31012701988220215, "kl": 3.84765625, "learning_rate": 1.0216446939474234e-06, "loss": 0.2164, "num_tokens": 1332725055.0, "reward": 0.6657366305589676, "reward_std": 0.2194537203758955, "rewards/accuracy_reward/mean": 0.18303571734577417, "rewards/accuracy_reward/std": 0.35302431136369705, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06349284294992685, "step": 2783 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 861.8348693847656, "completions/mean_terminated_length": 745.7860412597656, "completions/min_length": 290.0, "completions/min_terminated_length": 290.0, "epoch": 0.8316033156597715, "grad_norm": 0.2398572713136673, "kl": 1.7705078125, "learning_rate": 1.0168467708984097e-06, "loss": 0.0864, "num_tokens": 1333187525.0, "reward": 0.5904017984867096, "reward_std": 0.10546717792749405, "rewards/accuracy_reward/mean": 0.09821428684517741, "rewards/accuracy_reward/std": 0.2659534029662609, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04137531528249383, "step": 2784 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 847.3482513427734, "completions/mean_terminated_length": 728.3416595458984, "completions/min_length": 324.25, "completions/min_terminated_length": 324.25, "epoch": 0.8319020237472929, "grad_norm": 0.3546597957611084, "kl": 2.40625, "learning_rate": 1.012059537008332e-06, "loss": 0.1338, "num_tokens": 1333635169.0, "reward": 0.6562500298023224, "reward_std": 0.1529064141213894, "rewards/accuracy_reward/mean": 0.16741071082651615, "rewards/accuracy_reward/std": 0.3515682481229305, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.052474538795650005, "step": 2785 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 819.4219055175781, "completions/mean_terminated_length": 702.2162780761719, "completions/min_length": 292.25, "completions/min_terminated_length": 292.25, "epoch": 0.8322007318348145, "grad_norm": 0.3570218086242676, "kl": 3.5859375, "learning_rate": 1.0072829979735698e-06, "loss": 0.2038, "num_tokens": 1334080142.0, "reward": 0.6406250149011612, "reward_std": 0.18196208775043488, "rewards/accuracy_reward/mean": 0.15401785681024194, "rewards/accuracy_reward/std": 0.33243008330464363, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05643500294536352, "step": 2786 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 886.8281707763672, "completions/mean_terminated_length": 764.3456573486328, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "epoch": 0.8324994399223359, "grad_norm": 0.25723129510879517, "kl": 2.2421875, "learning_rate": 1.0025171594777872e-06, "loss": 0.1084, "num_tokens": 1334554561.0, "reward": 0.6071428954601288, "reward_std": 0.1508485432714224, "rewards/accuracy_reward/mean": 0.11607142840512097, "rewards/accuracy_reward/std": 0.2755946237593889, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04575194884091616, "step": 2787 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 839.7076263427734, "completions/mean_terminated_length": 712.2139434814453, "completions/min_length": 275.5, "completions/min_terminated_length": 275.5, "epoch": 0.8327981480098574, "grad_norm": 0.40044111013412476, "kl": 2.744140625, "learning_rate": 9.977620271919087e-07, "loss": 0.1402, "num_tokens": 1335002606.0, "reward": 0.6696428954601288, "reward_std": 0.1533821765333414, "rewards/accuracy_reward/mean": 0.1808035671710968, "rewards/accuracy_reward/std": 0.31765488535165787, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050504449754953384, "step": 2788 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 865.9620971679688, "completions/mean_terminated_length": 738.6065063476562, "completions/min_length": 359.25, "completions/min_terminated_length": 359.25, "epoch": 0.8330968560973788, "grad_norm": 0.4014976918697357, "kl": 2.87890625, "learning_rate": 9.930176067741216e-07, "loss": 0.142, "num_tokens": 1335463293.0, "reward": 0.7622767984867096, "reward_std": 0.15161005221307278, "rewards/accuracy_reward/mean": 0.276785708963871, "rewards/accuracy_reward/std": 0.447524718940258, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.06044707912951708, "step": 2789 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 828.700927734375, "completions/mean_terminated_length": 715.8186798095703, "completions/min_length": 250.0, "completions/min_terminated_length": 250.0, "epoch": 0.8333955641849004, "grad_norm": 0.33954426646232605, "kl": 3.52734375, "learning_rate": 9.882839038698688e-07, "loss": 0.1755, "num_tokens": 1335908295.0, "reward": 0.655133955180645, "reward_std": 0.17305949050933123, "rewards/accuracy_reward/mean": 0.1826636902987957, "rewards/accuracy_reward/std": 0.32118765264749527, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.06055070646107197, "step": 2790 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 839.8482513427734, "completions/mean_terminated_length": 737.922119140625, "completions/min_length": 262.75, "completions/min_terminated_length": 262.75, "epoch": 0.8336942722724218, "grad_norm": 0.20145434141159058, "kl": 1.65966796875, "learning_rate": 9.835609241118404e-07, "loss": 0.0907, "num_tokens": 1336362051.0, "reward": 0.5892857313156128, "reward_std": 0.11796863563358784, "rewards/accuracy_reward/mean": 0.09598214318975806, "rewards/accuracy_reward/std": 0.27411238104104996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03275321377441287, "step": 2791 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 886.8795013427734, "completions/mean_terminated_length": 775.972900390625, "completions/min_length": 391.75, "completions/min_terminated_length": 391.75, "epoch": 0.8339929803599433, "grad_norm": 0.3700476288795471, "kl": 2.5, "learning_rate": 9.78848673119961e-07, "loss": 0.1199, "num_tokens": 1336840429.0, "reward": 0.6289062798023224, "reward_std": 0.20763446018099785, "rewards/accuracy_reward/mean": 0.14285714365541935, "rewards/accuracy_reward/std": 0.33506667986512184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05715245008468628, "step": 2792 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 883.4933319091797, "completions/mean_terminated_length": 778.1809234619141, "completions/min_length": 407.5, "completions/min_terminated_length": 407.5, "epoch": 0.8342916884474647, "grad_norm": 0.3891286849975586, "kl": 2.34375, "learning_rate": 9.74147156501396e-07, "loss": 0.1167, "num_tokens": 1337307770.0, "reward": 0.6640625298023224, "reward_std": 0.19106141291558743, "rewards/accuracy_reward/mean": 0.17410714644938707, "rewards/accuracy_reward/std": 0.34703488647937775, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04838141333311796, "step": 2793 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 831.1942443847656, "completions/mean_terminated_length": 732.6582641601562, "completions/min_length": 321.5, "completions/min_terminated_length": 321.5, "epoch": 0.8345903965349862, "grad_norm": 0.2663252055644989, "kl": 2.693359375, "learning_rate": 9.694563798505319e-07, "loss": 0.1417, "num_tokens": 1337749265.0, "reward": 0.686941996216774, "reward_std": 0.16803811118006706, "rewards/accuracy_reward/mean": 0.1986607164144516, "rewards/accuracy_reward/std": 0.3937912955880165, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.0513117304071784, "step": 2794 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 837.372802734375, "completions/mean_terminated_length": 748.0718841552734, "completions/min_length": 333.5, "completions/min_terminated_length": 333.5, "epoch": 0.8348891046225076, "grad_norm": 0.39600464701652527, "kl": 2.716796875, "learning_rate": 9.647763487489815e-07, "loss": 0.1424, "num_tokens": 1338194456.0, "reward": 0.7271205633878708, "reward_std": 0.21473664045333862, "rewards/accuracy_reward/mean": 0.2388392905704677, "rewards/accuracy_reward/std": 0.38355959206819534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052888848818838596, "step": 2795 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 868.1763916015625, "completions/mean_terminated_length": 772.20654296875, "completions/min_length": 395.75, "completions/min_terminated_length": 395.75, "epoch": 0.8351878127100292, "grad_norm": 0.2649635672569275, "kl": 2.33203125, "learning_rate": 9.601070687655667e-07, "loss": 0.1087, "num_tokens": 1338657191.0, "reward": 0.6484375298023224, "reward_std": 0.11185843031853437, "rewards/accuracy_reward/mean": 0.16071428498253226, "rewards/accuracy_reward/std": 0.3162103593349457, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.052645014598965645, "step": 2796 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 826.0268096923828, "completions/mean_terminated_length": 724.6107177734375, "completions/min_length": 401.0, "completions/min_terminated_length": 401.0, "epoch": 0.8354865207975506, "grad_norm": 0.2826542258262634, "kl": 2.77734375, "learning_rate": 9.55448545456319e-07, "loss": 0.1627, "num_tokens": 1339091875.0, "reward": 0.6780134290456772, "reward_std": 0.2201879285275936, "rewards/accuracy_reward/mean": 0.1897321380674839, "rewards/accuracy_reward/std": 0.36250366643071175, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501998484135, "step": 2797 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 849.7723541259766, "completions/mean_terminated_length": 743.1485900878906, "completions/min_length": 256.75, "completions/min_terminated_length": 256.75, "epoch": 0.8357852288850721, "grad_norm": 0.2603449821472168, "kl": 2.27734375, "learning_rate": 9.508007843644718e-07, "loss": 0.1175, "num_tokens": 1339541453.0, "reward": 0.650669664144516, "reward_std": 0.17587271705269814, "rewards/accuracy_reward/mean": 0.16071428498253226, "rewards/accuracy_reward/std": 0.31533021852374077, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 2798 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39732142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 849.8460235595703, "completions/mean_terminated_length": 738.3572692871094, "completions/min_length": 318.0, "completions/min_terminated_length": 318.0, "epoch": 0.8360839369725935, "grad_norm": 0.2735302448272705, "kl": 2.25390625, "learning_rate": 9.461637910204468e-07, "loss": 0.119, "num_tokens": 1340000600.0, "reward": 0.6880580633878708, "reward_std": 0.14507286809384823, "rewards/accuracy_reward/mean": 0.19642857694998384, "rewards/accuracy_reward/std": 0.3573802076280117, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.043143877293914557, "step": 2799 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 813.7924346923828, "completions/mean_terminated_length": 684.7986297607422, "completions/min_length": 311.25, "completions/min_terminated_length": 311.25, "epoch": 0.836382645060115, "grad_norm": 0.41282305121421814, "kl": 3.533203125, "learning_rate": 9.415375709418606e-07, "loss": 0.2168, "num_tokens": 1340436859.0, "reward": 0.6718750447034836, "reward_std": 0.18925788113847375, "rewards/accuracy_reward/mean": 0.18750000558793545, "rewards/accuracy_reward/std": 0.2918257713317871, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.059125179424881935, "step": 2800 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31026785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 831.8995819091797, "completions/mean_terminated_length": 749.5624084472656, "completions/min_length": 433.75, "completions/min_terminated_length": 433.75, "epoch": 0.8366813531476365, "grad_norm": 0.40948933362960815, "kl": 2.21484375, "learning_rate": 9.369221296335007e-07, "loss": 0.1266, "num_tokens": 1340877422.0, "reward": 0.7137276977300644, "reward_std": 0.1666063256561756, "rewards/accuracy_reward/mean": 0.223214291036129, "rewards/accuracy_reward/std": 0.39551956951618195, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046612851321697235, "step": 2801 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47321428571428564, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 886.6696929931641, "completions/mean_terminated_length": 767.7139739990234, "completions/min_length": 394.75, "completions/min_terminated_length": 394.75, "epoch": 0.8369800612351579, "grad_norm": 0.28864580392837524, "kl": 2.8515625, "learning_rate": 9.323174725873407e-07, "loss": 0.1361, "num_tokens": 1341352538.0, "reward": 0.7003348469734192, "reward_std": 0.16697960533201694, "rewards/accuracy_reward/mean": 0.21428571455180645, "rewards/accuracy_reward/std": 0.4038286805152893, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05749546363949776, "step": 2802 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 867.4219055175781, "completions/mean_terminated_length": 759.3654479980469, "completions/min_length": 307.75, "completions/min_terminated_length": 307.75, "epoch": 0.8372787693226794, "grad_norm": 0.3728272020816803, "kl": 2.828125, "learning_rate": 9.277236052825078e-07, "loss": 0.1418, "num_tokens": 1341815207.0, "reward": 0.5820312798023224, "reward_std": 0.11981315352022648, "rewards/accuracy_reward/mean": 0.09598214249126613, "rewards/accuracy_reward/std": 0.24578144028782845, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05687962658703327, "step": 2803 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 854.1920013427734, "completions/mean_terminated_length": 745.5987396240234, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "epoch": 0.8375774774102008, "grad_norm": 0.33255550265312195, "kl": 1.6513671875, "learning_rate": 9.231405331853082e-07, "loss": 0.096, "num_tokens": 1342265485.0, "reward": 0.729910746216774, "reward_std": 0.12404085323214531, "rewards/accuracy_reward/mean": 0.23660714458674192, "rewards/accuracy_reward/std": 0.38515830785036087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03759844787418842, "step": 2804 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 831.3214721679688, "completions/mean_terminated_length": 709.5211486816406, "completions/min_length": 280.25, "completions/min_terminated_length": 280.25, "epoch": 0.8378761854977224, "grad_norm": 0.28777527809143066, "kl": 2.123046875, "learning_rate": 9.185682617491865e-07, "loss": 0.1125, "num_tokens": 1342708637.0, "reward": 0.5172991380095482, "reward_std": 0.060885097132995725, "rewards/accuracy_reward/mean": 0.02455357206054032, "rewards/accuracy_reward/std": 0.09523210488259792, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040314854588359594, "step": 2805 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 853.529052734375, "completions/mean_terminated_length": 732.1429138183594, "completions/min_length": 331.25, "completions/min_terminated_length": 331.25, "epoch": 0.8381748935852438, "grad_norm": 0.3336920738220215, "kl": 1.462890625, "learning_rate": 9.140067964147447e-07, "loss": 0.0756, "num_tokens": 1343170378.0, "reward": 0.6914062798023224, "reward_std": 0.17054973915219307, "rewards/accuracy_reward/mean": 0.19642857648432255, "rewards/accuracy_reward/std": 0.37663818895816803, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.03418479347601533, "step": 2806 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 831.1272583007812, "completions/mean_terminated_length": 705.5389709472656, "completions/min_length": 242.75, "completions/min_terminated_length": 242.75, "epoch": 0.8384736016727653, "grad_norm": 0.1802682727575302, "kl": 1.984375, "learning_rate": 9.094561426097215e-07, "loss": 0.0861, "num_tokens": 1343612595.0, "reward": 0.6428571790456772, "reward_std": 0.11866173520684242, "rewards/accuracy_reward/mean": 0.1640625, "rewards/accuracy_reward/std": 0.35548483207821846, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050577715039253235, "step": 2807 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3303571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 814.5915374755859, "completions/mean_terminated_length": 711.9407653808594, "completions/min_length": 336.75, "completions/min_terminated_length": 336.75, "epoch": 0.8387723097602867, "grad_norm": 0.3622584640979767, "kl": 2.23828125, "learning_rate": 9.049163057489963e-07, "loss": 0.1369, "num_tokens": 1344041916.0, "reward": 0.8102678954601288, "reward_std": 0.17543626762926579, "rewards/accuracy_reward/mean": 0.3169642835855484, "rewards/accuracy_reward/std": 0.4551929756999016, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 2808 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 839.0714721679688, "completions/mean_terminated_length": 738.1417846679688, "completions/min_length": 289.5, "completions/min_terminated_length": 289.5, "epoch": 0.8390710178478082, "grad_norm": 0.2697322964668274, "kl": 2.51171875, "learning_rate": 9.00387291234569e-07, "loss": 0.1367, "num_tokens": 1344490524.0, "reward": 0.6718750149011612, "reward_std": 0.17782128974795341, "rewards/accuracy_reward/mean": 0.18303571082651615, "rewards/accuracy_reward/std": 0.37220292538404465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05137455835938454, "step": 2809 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 826.6451416015625, "completions/mean_terminated_length": 724.3401184082031, "completions/min_length": 343.75, "completions/min_terminated_length": 343.75, "epoch": 0.8393697259353297, "grad_norm": 0.460183322429657, "kl": 2.935546875, "learning_rate": 8.95869104455569e-07, "loss": 0.1747, "num_tokens": 1344933021.0, "reward": 0.6506696790456772, "reward_std": 0.19653484970331192, "rewards/accuracy_reward/mean": 0.17038690112531185, "rewards/accuracy_reward/std": 0.3706243112683296, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.057508016005158424, "step": 2810 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 859.5937805175781, "completions/mean_terminated_length": 768.3818969726562, "completions/min_length": 382.75, "completions/min_terminated_length": 382.75, "epoch": 0.8396684340228512, "grad_norm": 0.27500051259994507, "kl": 2.54296875, "learning_rate": 8.91361750788241e-07, "loss": 0.1288, "num_tokens": 1345389207.0, "reward": 0.7059151977300644, "reward_std": 0.18025367334485054, "rewards/accuracy_reward/mean": 0.218750003259629, "rewards/accuracy_reward/std": 0.37027157098054886, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 2811 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 811.7098541259766, "completions/mean_terminated_length": 705.3982543945312, "completions/min_length": 279.75, "completions/min_terminated_length": 279.75, "epoch": 0.8399671421103726, "grad_norm": 0.3614000678062439, "kl": 2.59375, "learning_rate": 8.868652355959384e-07, "loss": 0.169, "num_tokens": 1345818165.0, "reward": 0.6422991305589676, "reward_std": 0.16029172576963902, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.34481725841760635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 2812 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 842.4152221679688, "completions/mean_terminated_length": 742.2489166259766, "completions/min_length": 350.0, "completions/min_terminated_length": 350.0, "epoch": 0.8402658501978941, "grad_norm": 0.39558711647987366, "kl": 2.439453125, "learning_rate": 8.823795642291145e-07, "loss": 0.1405, "num_tokens": 1346262239.0, "reward": 0.6372768208384514, "reward_std": 0.1339487051591277, "rewards/accuracy_reward/mean": 0.1495535746216774, "rewards/accuracy_reward/std": 0.29122401773929596, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 2813 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 858.4375305175781, "completions/mean_terminated_length": 744.7820129394531, "completions/min_length": 328.25, "completions/min_terminated_length": 328.25, "epoch": 0.8405645582854155, "grad_norm": 0.595695436000824, "kl": 3.546875, "learning_rate": 8.779047420253239e-07, "loss": 0.1762, "num_tokens": 1346718371.0, "reward": 0.659598246216774, "reward_std": 0.1904318518936634, "rewards/accuracy_reward/mean": 0.1893601194024086, "rewards/accuracy_reward/std": 0.3734513819217682, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4832589253783226, "rewards/tag_count_reward/std": 0.060757264494895935, "step": 2814 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 845.8482513427734, "completions/mean_terminated_length": 738.8140869140625, "completions/min_length": 365.75, "completions/min_terminated_length": 365.75, "epoch": 0.8408632663729371, "grad_norm": 0.20858007669448853, "kl": 2.216796875, "learning_rate": 8.734407743092078e-07, "loss": 0.1062, "num_tokens": 1347172239.0, "reward": 0.6261161118745804, "reward_std": 0.11665690876543522, "rewards/accuracy_reward/mean": 0.13616071734577417, "rewards/accuracy_reward/std": 0.3220384567975998, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04688958963379264, "step": 2815 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 822.1406707763672, "completions/mean_terminated_length": 712.6555786132812, "completions/min_length": 269.25, "completions/min_terminated_length": 269.25, "epoch": 0.8411619744604585, "grad_norm": 0.3742920160293579, "kl": 1.4560546875, "learning_rate": 8.689876663924957e-07, "loss": 0.0703, "num_tokens": 1347611374.0, "reward": 0.6640625447034836, "reward_std": 0.13970251567661762, "rewards/accuracy_reward/mean": 0.1696428614668548, "rewards/accuracy_reward/std": 0.3210551552474499, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.031923466362059116, "step": 2816 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39732142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 863.8616333007812, "completions/mean_terminated_length": 759.2099761962891, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "epoch": 0.84146068254798, "grad_norm": 0.23029354214668274, "kl": 2.623046875, "learning_rate": 8.645454235739903e-07, "loss": 0.1327, "num_tokens": 1348082288.0, "reward": 0.611607164144516, "reward_std": 0.1512665692716837, "rewards/accuracy_reward/mean": 0.12276785541325808, "rewards/accuracy_reward/std": 0.32052624970674515, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05112028680741787, "step": 2817 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 859.935302734375, "completions/mean_terminated_length": 757.0785675048828, "completions/min_length": 376.5, "completions/min_terminated_length": 376.5, "epoch": 0.8417593906355014, "grad_norm": 0.4888005554676056, "kl": 2.64453125, "learning_rate": 8.601140511395723e-07, "loss": 0.145, "num_tokens": 1348540835.0, "reward": 0.592075914144516, "reward_std": 0.16213861480355263, "rewards/accuracy_reward/mean": 0.10491071548312902, "rewards/accuracy_reward/std": 0.28905489295721054, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05292889382690191, "step": 2818 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39955357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 858.841552734375, "completions/mean_terminated_length": 751.6109008789062, "completions/min_length": 344.25, "completions/min_terminated_length": 344.25, "epoch": 0.842058098723023, "grad_norm": 0.3126711845397949, "kl": 1.763671875, "learning_rate": 8.556935543621791e-07, "loss": 0.0747, "num_tokens": 1348997436.0, "reward": 0.5781250298023224, "reward_std": 0.12381068919785321, "rewards/accuracy_reward/mean": 0.0870535708963871, "rewards/accuracy_reward/std": 0.22930337488651276, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04386132536455989, "step": 2819 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 853.2209930419922, "completions/mean_terminated_length": 754.978515625, "completions/min_length": 304.75, "completions/min_terminated_length": 304.75, "epoch": 0.8423568068105444, "grad_norm": 0.24689123034477234, "kl": 1.8857421875, "learning_rate": 8.512839385018146e-07, "loss": 0.1005, "num_tokens": 1349457535.0, "reward": 0.6138392984867096, "reward_std": 0.11594023834913969, "rewards/accuracy_reward/mean": 0.12053571501746774, "rewards/accuracy_reward/std": 0.2820524126291275, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 2820 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 872.2857666015625, "completions/mean_terminated_length": 768.8089447021484, "completions/min_length": 353.5, "completions/min_terminated_length": 353.5, "epoch": 0.8426555148980659, "grad_norm": 0.31967806816101074, "kl": 2.0888671875, "learning_rate": 8.468852088055291e-07, "loss": 0.1132, "num_tokens": 1349920623.0, "reward": 0.6540178954601288, "reward_std": 0.15849998407065868, "rewards/accuracy_reward/mean": 0.16294642654247582, "rewards/accuracy_reward/std": 0.3238630201667547, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.044658167753368616, "step": 2821 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 849.2098541259766, "completions/mean_terminated_length": 724.5874633789062, "completions/min_length": 354.0, "completions/min_terminated_length": 354.0, "epoch": 0.8429542229855873, "grad_norm": 0.5337969660758972, "kl": 3.3515625, "learning_rate": 8.424973705074258e-07, "loss": 0.1737, "num_tokens": 1350373117.0, "reward": 0.6880580633878708, "reward_std": 0.22510390728712082, "rewards/accuracy_reward/mean": 0.20089285634458065, "rewards/accuracy_reward/std": 0.39528557658195496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102820426226, "step": 2822 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 791.404052734375, "completions/mean_terminated_length": 684.7423706054688, "completions/min_length": 286.0, "completions/min_terminated_length": 286.0, "epoch": 0.8432529310731088, "grad_norm": 0.43559154868125916, "kl": 3.3984375, "learning_rate": 8.381204288286415e-07, "loss": 0.1957, "num_tokens": 1350794066.0, "reward": 0.741629496216774, "reward_std": 0.18253923207521439, "rewards/accuracy_reward/mean": 0.2730654841288924, "rewards/accuracy_reward/std": 0.4096934348344803, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05856847669929266, "step": 2823 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2879464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 798.7589569091797, "completions/mean_terminated_length": 706.0638885498047, "completions/min_length": 364.0, "completions/min_terminated_length": 364.0, "epoch": 0.8435516391606303, "grad_norm": 0.46744731068611145, "kl": 2.556640625, "learning_rate": 8.337543889773525e-07, "loss": 0.1305, "num_tokens": 1351222870.0, "reward": 0.5993303805589676, "reward_std": 0.1511731524951756, "rewards/accuracy_reward/mean": 0.10937500186264515, "rewards/accuracy_reward/std": 0.25132279098033905, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04716797545552254, "step": 2824 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3303571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 795.9464569091797, "completions/mean_terminated_length": 682.6519470214844, "completions/min_length": 253.5, "completions/min_terminated_length": 253.5, "epoch": 0.8438503472481518, "grad_norm": 0.28534412384033203, "kl": 1.998046875, "learning_rate": 8.293992561487596e-07, "loss": 0.113, "num_tokens": 1351650318.0, "reward": 0.7003348469734192, "reward_std": 0.18781277537345886, "rewards/accuracy_reward/mean": 0.2075892835855484, "rewards/accuracy_reward/std": 0.40119925141334534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196588039398, "step": 2825 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 823.5446624755859, "completions/mean_terminated_length": 697.9471588134766, "completions/min_length": 328.25, "completions/min_terminated_length": 328.25, "epoch": 0.8441490553356732, "grad_norm": 0.5622290372848511, "kl": 2.330078125, "learning_rate": 8.250550355250875e-07, "loss": 0.1638, "num_tokens": 1352087298.0, "reward": 0.6339285969734192, "reward_std": 0.1548478789627552, "rewards/accuracy_reward/mean": 0.15178571385331452, "rewards/accuracy_reward/std": 0.32006361708045006, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04660273157060146, "step": 2826 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 846.3259124755859, "completions/mean_terminated_length": 744.239013671875, "completions/min_length": 373.25, "completions/min_terminated_length": 373.25, "epoch": 0.8444477634231947, "grad_norm": 0.3813900947570801, "kl": 3.2265625, "learning_rate": 8.207217322755734e-07, "loss": 0.1829, "num_tokens": 1352538292.0, "reward": 0.7299107313156128, "reward_std": 0.21911263093352318, "rewards/accuracy_reward/mean": 0.24330356903374195, "rewards/accuracy_reward/std": 0.40229327976703644, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.055638475343585014, "step": 2827 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 813.5201263427734, "completions/mean_terminated_length": 685.9840393066406, "completions/min_length": 259.0, "completions/min_terminated_length": 259.0, "epoch": 0.8447464715107161, "grad_norm": 0.4103035628795624, "kl": 2.646484375, "learning_rate": 8.163993515564672e-07, "loss": 0.1487, "num_tokens": 1352973277.0, "reward": 0.641183078289032, "reward_std": 0.18469994142651558, "rewards/accuracy_reward/mean": 0.15178571362048388, "rewards/accuracy_reward/std": 0.33894238620996475, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 2828 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 895.1406707763672, "completions/mean_terminated_length": 798.4278411865234, "completions/min_length": 323.25, "completions/min_terminated_length": 323.25, "epoch": 0.8450451795982377, "grad_norm": 0.2839573919773102, "kl": 1.0634765625, "learning_rate": 8.120878985110181e-07, "loss": 0.0501, "num_tokens": 1353467100.0, "reward": 0.5954241305589676, "reward_std": 0.11898777820169926, "rewards/accuracy_reward/mean": 0.09821428474970162, "rewards/accuracy_reward/std": 0.2711102943867445, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4972098171710968, "rewards/tag_count_reward/std": 0.02253411104902625, "step": 2829 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 815.3236999511719, "completions/mean_terminated_length": 722.6441802978516, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.8453438876857591, "grad_norm": 0.29423126578330994, "kl": 2.07421875, "learning_rate": 8.077873782694745e-07, "loss": 0.1162, "num_tokens": 1353904541.0, "reward": 0.7237723469734192, "reward_std": 0.13401910522952676, "rewards/accuracy_reward/mean": 0.23511905409395695, "rewards/accuracy_reward/std": 0.3957463651895523, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 2830 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42187500000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 872.2745971679688, "completions/mean_terminated_length": 768.5396881103516, "completions/min_length": 402.0, "completions/min_terminated_length": 402.0, "epoch": 0.8456425957732806, "grad_norm": 0.1875162273645401, "kl": 1.8515625, "learning_rate": 8.034977959490775e-07, "loss": 0.1025, "num_tokens": 1354366584.0, "reward": 0.6261160969734192, "reward_std": 0.17878633551299572, "rewards/accuracy_reward/mean": 0.13392857275903225, "rewards/accuracy_reward/std": 0.3323168456554413, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2831 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 816.7991180419922, "completions/mean_terminated_length": 708.4058685302734, "completions/min_length": 266.25, "completions/min_terminated_length": 266.25, "epoch": 0.845941303860802, "grad_norm": 0.34736815094947815, "kl": 1.974609375, "learning_rate": 7.992191566540519e-07, "loss": 0.1143, "num_tokens": 1354807534.0, "reward": 0.6573661118745804, "reward_std": 0.15417616441845894, "rewards/accuracy_reward/mean": 0.16517856903374195, "rewards/accuracy_reward/std": 0.3607931509613991, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04357414972037077, "step": 2832 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 845.6004943847656, "completions/mean_terminated_length": 734.591552734375, "completions/min_length": 322.25, "completions/min_terminated_length": 322.25, "epoch": 0.8462400119483235, "grad_norm": 0.2420406937599182, "kl": 1.5078125, "learning_rate": 7.949514654755963e-07, "loss": 0.0635, "num_tokens": 1355248859.0, "reward": 0.7154018133878708, "reward_std": 0.1488836221396923, "rewards/accuracy_reward/mean": 0.22098213993012905, "rewards/accuracy_reward/std": 0.37785395607352257, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 2833 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 824.3772735595703, "completions/mean_terminated_length": 710.8049774169922, "completions/min_length": 323.0, "completions/min_terminated_length": 323.0, "epoch": 0.846538720035845, "grad_norm": 0.17537613213062286, "kl": 0.9560546875, "learning_rate": 7.906947274918919e-07, "loss": 0.0599, "num_tokens": 1355685908.0, "reward": 0.7170759290456772, "reward_std": 0.10931890644133091, "rewards/accuracy_reward/mean": 0.22098213993012905, "rewards/accuracy_reward/std": 0.4091252163052559, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.49609375, "rewards/tag_count_reward/std": 0.025870586279779673, "step": 2834 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4352678571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 869.1629791259766, "completions/mean_terminated_length": 746.5007781982422, "completions/min_length": 282.25, "completions/min_terminated_length": 282.25, "epoch": 0.8468374281233665, "grad_norm": 0.43733885884284973, "kl": 2.57421875, "learning_rate": 7.864489477680759e-07, "loss": 0.1326, "num_tokens": 1356153037.0, "reward": 0.672433078289032, "reward_std": 0.1266104318201542, "rewards/accuracy_reward/mean": 0.1830357164144516, "rewards/accuracy_reward/std": 0.3225128948688507, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932562112808, "step": 2835 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 837.2879791259766, "completions/mean_terminated_length": 700.7872924804688, "completions/min_length": 349.75, "completions/min_terminated_length": 349.75, "epoch": 0.8471361362108879, "grad_norm": 0.3608911633491516, "kl": 2.19140625, "learning_rate": 7.822141313562548e-07, "loss": 0.1331, "num_tokens": 1356600590.0, "reward": 0.569196455180645, "reward_std": 0.14851090405136347, "rewards/accuracy_reward/mean": 0.078125, "rewards/accuracy_reward/std": 0.23007787764072418, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317149460316, "step": 2836 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 866.2433471679688, "completions/mean_terminated_length": 758.2559509277344, "completions/min_length": 295.5, "completions/min_terminated_length": 295.5, "epoch": 0.8474348442984094, "grad_norm": 0.4672392010688782, "kl": 2.791015625, "learning_rate": 7.779902832954833e-07, "loss": 0.1342, "num_tokens": 1357062043.0, "reward": 0.600446455180645, "reward_std": 0.1366987214423716, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.20738685131072998, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05030489154160023, "step": 2837 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 870.9888610839844, "completions/mean_terminated_length": 777.1640167236328, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.8477335523859308, "grad_norm": 0.19944120943546295, "kl": 1.90234375, "learning_rate": 7.737774086117678e-07, "loss": 0.1045, "num_tokens": 1357531878.0, "reward": 0.6997768133878708, "reward_std": 0.16485285572707653, "rewards/accuracy_reward/mean": 0.20758928172290325, "rewards/accuracy_reward/std": 0.3940107747912407, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04182914597913623, "step": 2838 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 857.247802734375, "completions/mean_terminated_length": 750.29443359375, "completions/min_length": 231.75, "completions/min_terminated_length": 231.75, "epoch": 0.8480322604734524, "grad_norm": 0.6705541610717773, "kl": 2.7421875, "learning_rate": 7.695755123180593e-07, "loss": 0.119, "num_tokens": 1357986741.0, "reward": 0.713169664144516, "reward_std": 0.16914345882833004, "rewards/accuracy_reward/mean": 0.22544643469154835, "rewards/accuracy_reward/std": 0.3893963694572449, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05333347246050835, "step": 2839 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 883.5826263427734, "completions/mean_terminated_length": 765.4417877197266, "completions/min_length": 397.5, "completions/min_terminated_length": 397.5, "epoch": 0.8483309685609738, "grad_norm": 0.25322484970092773, "kl": 2.4296875, "learning_rate": 7.653845994142428e-07, "loss": 0.1228, "num_tokens": 1358456842.0, "reward": 0.6127232313156128, "reward_std": 0.17015660740435123, "rewards/accuracy_reward/mean": 0.12276785913854837, "rewards/accuracy_reward/std": 0.3157297447323799, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04716797545552254, "step": 2840 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 829.7500305175781, "completions/mean_terminated_length": 737.7899322509766, "completions/min_length": 335.25, "completions/min_terminated_length": 335.25, "epoch": 0.8486296766484953, "grad_norm": 0.5094508528709412, "kl": 1.884765625, "learning_rate": 7.612046748871327e-07, "loss": 0.1228, "num_tokens": 1358894762.0, "reward": 0.6796875447034836, "reward_std": 0.19461626186966896, "rewards/accuracy_reward/mean": 0.1953125037252903, "rewards/accuracy_reward/std": 0.3840436786413193, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2841 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 849.9643249511719, "completions/mean_terminated_length": 720.3072509765625, "completions/min_length": 380.5, "completions/min_terminated_length": 380.5, "epoch": 0.8489283847360167, "grad_norm": 0.36298656463623047, "kl": 2.67578125, "learning_rate": 7.570357437104714e-07, "loss": 0.1472, "num_tokens": 1359356938.0, "reward": 0.7226562649011612, "reward_std": 0.20127731189131737, "rewards/accuracy_reward/mean": 0.23437500186264515, "rewards/accuracy_reward/std": 0.3794763833284378, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 2842 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 858.9330749511719, "completions/mean_terminated_length": 758.9544372558594, "completions/min_length": 418.0, "completions/min_terminated_length": 418.0, "epoch": 0.8492270928235381, "grad_norm": 0.27525225281715393, "kl": 2.26953125, "learning_rate": 7.528778108449197e-07, "loss": 0.1223, "num_tokens": 1359811484.0, "reward": 0.6545759290456772, "reward_std": 0.1567373387515545, "rewards/accuracy_reward/mean": 0.16517856949940324, "rewards/accuracy_reward/std": 0.33495279029011726, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973097205162, "rewards/tag_count_reward/std": 0.04783148504793644, "step": 2843 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 854.7344207763672, "completions/mean_terminated_length": 724.7465209960938, "completions/min_length": 293.25, "completions/min_terminated_length": 293.25, "epoch": 0.8495258009110597, "grad_norm": 0.2908867597579956, "kl": 2.525390625, "learning_rate": 7.487308812380467e-07, "loss": 0.134, "num_tokens": 1360260309.0, "reward": 0.7042410969734192, "reward_std": 0.17524918355047703, "rewards/accuracy_reward/mean": 0.2142857122235, "rewards/accuracy_reward/std": 0.36394202709198, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 2844 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 847.4397735595703, "completions/mean_terminated_length": 708.5289001464844, "completions/min_length": 334.5, "completions/min_terminated_length": 334.5, "epoch": 0.8498245089985811, "grad_norm": 0.2840782105922699, "kl": 2.205078125, "learning_rate": 7.445949598243362e-07, "loss": 0.13, "num_tokens": 1360717178.0, "reward": 0.5792410969734192, "reward_std": 0.10122289881110191, "rewards/accuracy_reward/mean": 0.08705357392318547, "rewards/accuracy_reward/std": 0.2560849077999592, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2845 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5178571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 913.3437957763672, "completions/mean_terminated_length": 793.093017578125, "completions/min_length": 380.5, "completions/min_terminated_length": 380.5, "epoch": 0.8501232170861026, "grad_norm": 0.30662304162979126, "kl": 2.58203125, "learning_rate": 7.404700515251672e-07, "loss": 0.1131, "num_tokens": 1361196884.0, "reward": 0.6049107313156128, "reward_std": 0.15079726767726243, "rewards/accuracy_reward/mean": 0.1183035746216774, "rewards/accuracy_reward/std": 0.3203268125653267, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05180213740095496, "step": 2846 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 830.1696624755859, "completions/mean_terminated_length": 723.0453491210938, "completions/min_length": 357.25, "completions/min_terminated_length": 357.25, "epoch": 0.850421925173624, "grad_norm": 0.22780050337314606, "kl": 1.97412109375, "learning_rate": 7.363561612488191e-07, "loss": 0.1216, "num_tokens": 1361641152.0, "reward": 0.7912946790456772, "reward_std": 0.20984485745429993, "rewards/accuracy_reward/mean": 0.2991071417927742, "rewards/accuracy_reward/std": 0.45364201068878174, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2847 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 848.0111846923828, "completions/mean_terminated_length": 727.7485504150391, "completions/min_length": 351.25, "completions/min_terminated_length": 351.25, "epoch": 0.8507206332611456, "grad_norm": 0.30604323744773865, "kl": 2.5078125, "learning_rate": 7.322532938904548e-07, "loss": 0.1383, "num_tokens": 1362099429.0, "reward": 0.7360491454601288, "reward_std": 0.16860169731080532, "rewards/accuracy_reward/mean": 0.24776785634458065, "rewards/accuracy_reward/std": 0.4093998447060585, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052180747501552105, "step": 2848 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36383928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 840.1830749511719, "completions/mean_terminated_length": 740.1078491210938, "completions/min_length": 373.25, "completions/min_terminated_length": 373.25, "epoch": 0.851019341348667, "grad_norm": 0.2351624071598053, "kl": 1.447265625, "learning_rate": 7.281614543321269e-07, "loss": 0.0684, "num_tokens": 1362547175.0, "reward": 0.5781250149011612, "reward_std": 0.04693942470476031, "rewards/accuracy_reward/mean": 0.0825892873108387, "rewards/accuracy_reward/std": 0.18589160591363907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 2849 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33035714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 838.1719207763672, "completions/mean_terminated_length": 756.2450103759766, "completions/min_length": 336.25, "completions/min_terminated_length": 336.25, "epoch": 0.8513180494361885, "grad_norm": 0.2378934770822525, "kl": 1.87109375, "learning_rate": 7.240806474427598e-07, "loss": 0.0986, "num_tokens": 1362991028.0, "reward": 0.6914062947034836, "reward_std": 0.15500010177493095, "rewards/accuracy_reward/mean": 0.2083333358168602, "rewards/accuracy_reward/std": 0.4032198637723923, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 2850 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 833.9977874755859, "completions/mean_terminated_length": 725.8579864501953, "completions/min_length": 357.0, "completions/min_terminated_length": 357.0, "epoch": 0.8516167575237099, "grad_norm": 0.42994385957717896, "kl": 2.046875, "learning_rate": 7.200108780781556e-07, "loss": 0.127, "num_tokens": 1363436019.0, "reward": 0.710379496216774, "reward_std": 0.16221073642373085, "rewards/accuracy_reward/mean": 0.22358630783855915, "rewards/accuracy_reward/std": 0.3780982792377472, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2851 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 832.0491485595703, "completions/mean_terminated_length": 701.0130004882812, "completions/min_length": 353.75, "completions/min_terminated_length": 353.75, "epoch": 0.8519154656112314, "grad_norm": 0.2782895267009735, "kl": 1.8173828125, "learning_rate": 7.159521510809797e-07, "loss": 0.1078, "num_tokens": 1363880793.0, "reward": 0.6523437798023224, "reward_std": 0.1384190171957016, "rewards/accuracy_reward/mean": 0.16071428591385484, "rewards/accuracy_reward/std": 0.33680153265595436, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04293336346745491, "step": 2852 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 845.716552734375, "completions/mean_terminated_length": 742.9952239990234, "completions/min_length": 297.75, "completions/min_terminated_length": 297.75, "epoch": 0.8522141736987529, "grad_norm": 0.18814416229724884, "kl": 2.205078125, "learning_rate": 7.119044712807577e-07, "loss": 0.1117, "num_tokens": 1364339178.0, "reward": 0.6657366454601288, "reward_std": 0.1255156211555004, "rewards/accuracy_reward/mean": 0.1819196417927742, "rewards/accuracy_reward/std": 0.3225144296884537, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.044580988585948944, "step": 2853 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 814.7500305175781, "completions/mean_terminated_length": 727.9861602783203, "completions/min_length": 305.25, "completions/min_terminated_length": 305.25, "epoch": 0.8525128817862744, "grad_norm": 0.27013593912124634, "kl": 2.1484375, "learning_rate": 7.078678434938724e-07, "loss": 0.1127, "num_tokens": 1364773690.0, "reward": 0.7003348618745804, "reward_std": 0.1790638603270054, "rewards/accuracy_reward/mean": 0.2098214253783226, "rewards/accuracy_reward/std": 0.40354791283607483, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04791746195405722, "step": 2854 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 831.9955749511719, "completions/mean_terminated_length": 736.1850738525391, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "epoch": 0.8528115898737958, "grad_norm": 0.3571362793445587, "kl": 2.82421875, "learning_rate": 7.038422725235561e-07, "loss": 0.1588, "num_tokens": 1365229432.0, "reward": 0.6556919813156128, "reward_std": 0.16274658776819706, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.37485408782958984, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.055270818062126637, "step": 2855 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 845.5625305175781, "completions/mean_terminated_length": 742.4366149902344, "completions/min_length": 423.25, "completions/min_terminated_length": 423.25, "epoch": 0.8531102979613173, "grad_norm": 0.31208473443984985, "kl": 1.76171875, "learning_rate": 6.998277631598793e-07, "loss": 0.0894, "num_tokens": 1365685956.0, "reward": 0.5881696715950966, "reward_std": 0.0891219568438828, "rewards/accuracy_reward/mean": 0.09598214295692742, "rewards/accuracy_reward/std": 0.2188947480171919, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.03960080398246646, "step": 2856 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5089285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 893.7812957763672, "completions/mean_terminated_length": 760.064208984375, "completions/min_length": 222.0, "completions/min_terminated_length": 222.0, "epoch": 0.8534090060488387, "grad_norm": 0.27926337718963623, "kl": 1.86328125, "learning_rate": 6.958243201797554e-07, "loss": 0.0741, "num_tokens": 1366167842.0, "reward": 0.608816996216774, "reward_std": 0.09776824526488781, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.27068452537059784, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04760421719402075, "step": 2857 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40401785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 864.8125457763672, "completions/mean_terminated_length": 759.1104583740234, "completions/min_length": 358.5, "completions/min_terminated_length": 358.5, "epoch": 0.8537077141363603, "grad_norm": 0.40866532921791077, "kl": 2.123046875, "learning_rate": 6.918319483469272e-07, "loss": 0.1265, "num_tokens": 1366628206.0, "reward": 0.5686384290456772, "reward_std": 0.13820492289960384, "rewards/accuracy_reward/mean": 0.07812500232830644, "rewards/accuracy_reward/std": 0.24597129598259926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.04672335181385279, "step": 2858 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 830.8214569091797, "completions/mean_terminated_length": 745.29248046875, "completions/min_length": 363.75, "completions/min_terminated_length": 363.75, "epoch": 0.8540064222238817, "grad_norm": 0.25399351119995117, "kl": 2.919921875, "learning_rate": 6.878506524119644e-07, "loss": 0.1597, "num_tokens": 1367073214.0, "reward": 0.6763393133878708, "reward_std": 0.14866289868950844, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.392928309738636, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05514051578938961, "step": 2859 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 866.5446929931641, "completions/mean_terminated_length": 744.8648376464844, "completions/min_length": 297.5, "completions/min_terminated_length": 297.5, "epoch": 0.8543051303114032, "grad_norm": 0.5281911492347717, "kl": 2.5859375, "learning_rate": 6.838804371122588e-07, "loss": 0.1181, "num_tokens": 1367530594.0, "reward": 0.6199777126312256, "reward_std": 0.18087736144661903, "rewards/accuracy_reward/mean": 0.1409970219247043, "rewards/accuracy_reward/std": 0.31339944899082184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574134543538094, "step": 2860 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 847.7969207763672, "completions/mean_terminated_length": 754.8225250244141, "completions/min_length": 390.75, "completions/min_terminated_length": 390.75, "epoch": 0.8546038383989246, "grad_norm": 0.3905694782733917, "kl": 2.34765625, "learning_rate": 6.799213071720156e-07, "loss": 0.1178, "num_tokens": 1367980119.0, "reward": 0.675781287252903, "reward_std": 0.1683282982558012, "rewards/accuracy_reward/mean": 0.18526785005815327, "rewards/accuracy_reward/std": 0.3353141527622938, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133843421936, "rewards/tag_count_reward/std": 0.044667141512036324, "step": 2861 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 816.8594055175781, "completions/mean_terminated_length": 693.8662109375, "completions/min_length": 376.5, "completions/min_terminated_length": 376.5, "epoch": 0.8549025464864461, "grad_norm": 0.25948581099510193, "kl": 1.78125, "learning_rate": 6.759732673022479e-07, "loss": 0.0975, "num_tokens": 1368412056.0, "reward": 0.5976562798023224, "reward_std": 0.1260928635019809, "rewards/accuracy_reward/mean": 0.10491071175783873, "rewards/accuracy_reward/std": 0.2382485345005989, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04000696213915944, "step": 2862 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 857.4464721679688, "completions/mean_terminated_length": 749.8770751953125, "completions/min_length": 383.75, "completions/min_terminated_length": 383.75, "epoch": 0.8552012545739676, "grad_norm": 0.2013280987739563, "kl": 1.39453125, "learning_rate": 6.720363222007786e-07, "loss": 0.0799, "num_tokens": 1368861536.0, "reward": 0.671316996216774, "reward_std": 0.17977612279355526, "rewards/accuracy_reward/mean": 0.17633928637951612, "rewards/accuracy_reward/std": 0.3649286553263664, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.034184794407337904, "step": 2863 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 861.6540679931641, "completions/mean_terminated_length": 768.4892120361328, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.8554999626614891, "grad_norm": 0.24816995859146118, "kl": 2.02734375, "learning_rate": 6.681104765522195e-07, "loss": 0.1102, "num_tokens": 1369320133.0, "reward": 0.6456473618745804, "reward_std": 0.16787764057517052, "rewards/accuracy_reward/mean": 0.15401785634458065, "rewards/accuracy_reward/std": 0.34696877375245094, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04488888196647167, "step": 2864 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 849.7522735595703, "completions/mean_terminated_length": 754.9754791259766, "completions/min_length": 447.25, "completions/min_terminated_length": 447.25, "epoch": 0.8557986707490105, "grad_norm": 0.25002846121788025, "kl": 1.0947265625, "learning_rate": 6.641957350279838e-07, "loss": 0.0666, "num_tokens": 1369775606.0, "reward": 0.5965401977300644, "reward_std": 0.08953387592919171, "rewards/accuracy_reward/mean": 0.10305059258826077, "rewards/accuracy_reward/std": 0.20707779750227928, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4960937574505806, "rewards/tag_count_reward/std": 0.029367766808718443, "step": 2865 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 849.2790679931641, "completions/mean_terminated_length": 751.0372924804688, "completions/min_length": 427.5, "completions/min_terminated_length": 427.5, "epoch": 0.856097378836532, "grad_norm": 0.3925573527812958, "kl": 1.763671875, "learning_rate": 6.602921022862663e-07, "loss": 0.0912, "num_tokens": 1370228419.0, "reward": 0.6651785969734192, "reward_std": 0.12871271558105946, "rewards/accuracy_reward/mean": 0.1741071380674839, "rewards/accuracy_reward/std": 0.3753082603216171, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04420433798804879, "step": 2866 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 855.982177734375, "completions/mean_terminated_length": 746.5095062255859, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "epoch": 0.8563960869240534, "grad_norm": 0.30253347754478455, "kl": 2.373046875, "learning_rate": 6.563995829720449e-07, "loss": 0.1275, "num_tokens": 1370686395.0, "reward": 0.5937500298023224, "reward_std": 0.1408700793981552, "rewards/accuracy_reward/mean": 0.1049107126891613, "rewards/accuracy_reward/std": 0.28962908685207367, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.0499969981610775, "step": 2867 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 997.75, "completions/mean_length": 850.9442291259766, "completions/mean_terminated_length": 744.6360931396484, "completions/min_length": 263.0, "completions/min_terminated_length": 263.0, "epoch": 0.856694795011575, "grad_norm": 0.34532082080841064, "kl": 1.826171875, "learning_rate": 6.525181817170756e-07, "loss": 0.0932, "num_tokens": 1371145506.0, "reward": 0.6590401977300644, "reward_std": 0.15023057535290718, "rewards/accuracy_reward/mean": 0.16741071944124997, "rewards/accuracy_reward/std": 0.28874088265001774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 2868 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 876.1629791259766, "completions/mean_terminated_length": 767.7442169189453, "completions/min_length": 429.25, "completions/min_terminated_length": 429.25, "epoch": 0.8569935030990964, "grad_norm": 0.24458789825439453, "kl": 1.9609375, "learning_rate": 6.48647903139884e-07, "loss": 0.0936, "num_tokens": 1371613899.0, "reward": 0.5803571790456772, "reward_std": 0.11835254728794098, "rewards/accuracy_reward/mean": 0.08928571408614516, "rewards/accuracy_reward/std": 0.2657776214182377, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04420433798804879, "step": 2869 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 872.8616485595703, "completions/mean_terminated_length": 785.644287109375, "completions/min_length": 390.75, "completions/min_terminated_length": 390.75, "epoch": 0.8572922111866179, "grad_norm": 0.228888601064682, "kl": 2.07421875, "learning_rate": 6.447887518457563e-07, "loss": 0.1143, "num_tokens": 1372081277.0, "reward": 0.6891741454601288, "reward_std": 0.21226399764418602, "rewards/accuracy_reward/mean": 0.1986607164144516, "rewards/accuracy_reward/std": 0.3809572644531727, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 2870 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 820.888427734375, "completions/mean_terminated_length": 727.8506622314453, "completions/min_length": 295.25, "completions/min_terminated_length": 295.25, "epoch": 0.8575909192741393, "grad_norm": 0.3329715430736542, "kl": 1.669921875, "learning_rate": 6.409407324267448e-07, "loss": 0.0807, "num_tokens": 1372523435.0, "reward": 0.6875000149011612, "reward_std": 0.1969618797302246, "rewards/accuracy_reward/mean": 0.19419642817229033, "rewards/accuracy_reward/std": 0.3757574111223221, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03849267074838281, "step": 2871 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 844.8616485595703, "completions/mean_terminated_length": 726.2684783935547, "completions/min_length": 276.5, "completions/min_terminated_length": 276.5, "epoch": 0.8578896273616609, "grad_norm": 0.2817012369632721, "kl": 2.7109375, "learning_rate": 6.371038494616488e-07, "loss": 0.1538, "num_tokens": 1372975437.0, "reward": 0.7126116454601288, "reward_std": 0.17911502346396446, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.41389667242765427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2872 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 882.5714569091797, "completions/mean_terminated_length": 787.128173828125, "completions/min_length": 356.75, "completions/min_terminated_length": 356.75, "epoch": 0.8581883354491823, "grad_norm": 0.21218672394752502, "kl": 3.068359375, "learning_rate": 6.332781075160244e-07, "loss": 0.1607, "num_tokens": 1373448141.0, "reward": 0.6512277126312256, "reward_std": 0.17999698221683502, "rewards/accuracy_reward/mean": 0.16517857229337096, "rewards/accuracy_reward/std": 0.3179489076137543, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05506143160164356, "step": 2873 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 844.2991485595703, "completions/mean_terminated_length": 706.2365417480469, "completions/min_length": 289.0, "completions/min_terminated_length": 289.0, "epoch": 0.8584870435367038, "grad_norm": 0.3861355781555176, "kl": 2.53515625, "learning_rate": 6.294635111421643e-07, "loss": 0.129, "num_tokens": 1373895635.0, "reward": 0.5904018133878708, "reward_std": 0.12454060651361942, "rewards/accuracy_reward/mean": 0.10044643003493547, "rewards/accuracy_reward/std": 0.2922644466161728, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886585831642, "step": 2874 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2589285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 816.8683471679688, "completions/mean_terminated_length": 745.05517578125, "completions/min_length": 279.0, "completions/min_terminated_length": 279.0, "epoch": 0.8587857516242252, "grad_norm": 0.24289222061634064, "kl": 1.876953125, "learning_rate": 6.256600648791034e-07, "loss": 0.0826, "num_tokens": 1374336984.0, "reward": 0.573660746216774, "reward_std": 0.12981299869716167, "rewards/accuracy_reward/mean": 0.08258928800933063, "rewards/accuracy_reward/std": 0.23722214810550213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 2875 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 861.2544860839844, "completions/mean_terminated_length": 761.1645355224609, "completions/min_length": 320.75, "completions/min_terminated_length": 320.75, "epoch": 0.8590844597117467, "grad_norm": 0.32704922556877136, "kl": 2.353515625, "learning_rate": 6.218677732526035e-07, "loss": 0.1204, "num_tokens": 1374797562.0, "reward": 0.6607143059372902, "reward_std": 0.14319498860277236, "rewards/accuracy_reward/mean": 0.1696428544819355, "rewards/accuracy_reward/std": 0.31023383885622025, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 2876 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 864.0067443847656, "completions/mean_terminated_length": 755.6521606445312, "completions/min_length": 350.25, "completions/min_terminated_length": 350.25, "epoch": 0.8593831677992682, "grad_norm": 0.3907405138015747, "kl": 1.861328125, "learning_rate": 6.180866407751595e-07, "loss": 0.0984, "num_tokens": 1375254605.0, "reward": 0.7198660969734192, "reward_std": 0.13974758051335812, "rewards/accuracy_reward/mean": 0.2276785746216774, "rewards/accuracy_reward/std": 0.3983435332775116, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043266257271170616, "step": 2877 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 858.3594207763672, "completions/mean_terminated_length": 735.2910308837891, "completions/min_length": 260.5, "completions/min_terminated_length": 260.5, "epoch": 0.8596818758867897, "grad_norm": 0.46024009585380554, "kl": 2.353515625, "learning_rate": 6.143166719459837e-07, "loss": 0.128, "num_tokens": 1375710750.0, "reward": 0.6858259290456772, "reward_std": 0.1968044899404049, "rewards/accuracy_reward/mean": 0.19642857392318547, "rewards/accuracy_reward/std": 0.31725248508155346, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2878 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 827.5469207763672, "completions/mean_terminated_length": 717.3867492675781, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "epoch": 0.8599805839743111, "grad_norm": 0.2462335228919983, "kl": 2.013671875, "learning_rate": 6.105578712510074e-07, "loss": 0.1189, "num_tokens": 1376163251.0, "reward": 0.651785746216774, "reward_std": 0.15733007714152336, "rewards/accuracy_reward/mean": 0.1607142798602581, "rewards/accuracy_reward/std": 0.3039059713482857, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 2879 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 826.3326263427734, "completions/mean_terminated_length": 740.3806304931641, "completions/min_length": 381.75, "completions/min_terminated_length": 381.75, "epoch": 0.8602792920618326, "grad_norm": 0.23652112483978271, "kl": 2.11328125, "learning_rate": 6.068102431628675e-07, "loss": 0.1138, "num_tokens": 1376607016.0, "reward": 0.6489955633878708, "reward_std": 0.16837787441909313, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.36599602550268173, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 2880 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 991.5, "completions/mean_length": 836.2098541259766, "completions/mean_terminated_length": 703.5202484130859, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.860578000149354, "grad_norm": 0.6164644956588745, "kl": 2.5390625, "learning_rate": 6.030737921409169e-07, "loss": 0.166, "num_tokens": 1377057286.0, "reward": 0.6914062798023224, "reward_std": 0.16166628152132034, "rewards/accuracy_reward/mean": 0.2031249962747097, "rewards/accuracy_reward/std": 0.344753697514534, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.052092005498707294, "step": 2881 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4910714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 889.6094055175781, "completions/mean_terminated_length": 757.0427856445312, "completions/min_length": 392.75, "completions/min_terminated_length": 392.75, "epoch": 0.8608767082368756, "grad_norm": 0.23314598202705383, "kl": 1.876953125, "learning_rate": 5.993485226311968e-07, "loss": 0.0911, "num_tokens": 1377531799.0, "reward": 0.6110491305589676, "reward_std": 0.1370931714773178, "rewards/accuracy_reward/mean": 0.1183035708963871, "rewards/accuracy_reward/std": 0.31593987718224525, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818479284644, "step": 2882 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.23660714285714288, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 803.2433319091797, "completions/mean_terminated_length": 735.3185729980469, "completions/min_length": 297.5, "completions/min_terminated_length": 297.5, "epoch": 0.861175416324397, "grad_norm": 0.3236232101917267, "kl": 2.08203125, "learning_rate": 5.956344390664525e-07, "loss": 0.1215, "num_tokens": 1377964692.0, "reward": 0.6796875298023224, "reward_std": 0.1827438324689865, "rewards/accuracy_reward/mean": 0.1874999962747097, "rewards/accuracy_reward/std": 0.39016813784837723, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04272336792200804, "step": 2883 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 827.7299346923828, "completions/mean_terminated_length": 731.0709686279297, "completions/min_length": 345.0, "completions/min_terminated_length": 345.0, "epoch": 0.8614741244119185, "grad_norm": 0.4135926365852356, "kl": 2.76953125, "learning_rate": 5.919315458661123e-07, "loss": 0.1742, "num_tokens": 1378403979.0, "reward": 0.7885045111179352, "reward_std": 0.23921975120902061, "rewards/accuracy_reward/mean": 0.2991071380674839, "rewards/accuracy_reward/std": 0.4501648098230362, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.489397332072258, "rewards/tag_count_reward/std": 0.05054692644625902, "step": 2884 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31026785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 819.9174499511719, "completions/mean_terminated_length": 729.26025390625, "completions/min_length": 333.25, "completions/min_terminated_length": 333.25, "epoch": 0.8617728324994399, "grad_norm": 0.3268062174320221, "kl": 2.70703125, "learning_rate": 5.882398474362949e-07, "loss": 0.1646, "num_tokens": 1378847238.0, "reward": 0.7829241305589676, "reward_std": 0.22208738327026367, "rewards/accuracy_reward/mean": 0.29464286006987095, "rewards/accuracy_reward/std": 0.4025563970208168, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05551016051322222, "step": 2885 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 841.8214569091797, "completions/mean_terminated_length": 735.2187957763672, "completions/min_length": 311.0, "completions/min_terminated_length": 311.0, "epoch": 0.8620715405869613, "grad_norm": 0.36708012223243713, "kl": 1.919921875, "learning_rate": 5.845593481697931e-07, "loss": 0.1097, "num_tokens": 1379298566.0, "reward": 0.6735491454601288, "reward_std": 0.19746297597885132, "rewards/accuracy_reward/mean": 0.18080357578583062, "rewards/accuracy_reward/std": 0.3416980840265751, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 2886 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 835.966552734375, "completions/mean_terminated_length": 725.121337890625, "completions/min_length": 393.75, "completions/min_terminated_length": 393.75, "epoch": 0.8623702486744829, "grad_norm": 0.20446477830410004, "kl": 1.8369140625, "learning_rate": 5.80890052446077e-07, "loss": 0.0945, "num_tokens": 1379746919.0, "reward": 0.6238839626312256, "reward_std": 0.09783595241606236, "rewards/accuracy_reward/mean": 0.13169642840512097, "rewards/accuracy_reward/std": 0.28411279805004597, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 2887 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3169642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 804.6027221679688, "completions/mean_terminated_length": 706.9470520019531, "completions/min_length": 384.75, "completions/min_terminated_length": 384.75, "epoch": 0.8626689567620043, "grad_norm": 0.2327774465084076, "kl": 1.623046875, "learning_rate": 5.772319646312841e-07, "loss": 0.0936, "num_tokens": 1380182149.0, "reward": 0.5965401977300644, "reward_std": 0.12091240705922246, "rewards/accuracy_reward/mean": 0.1097470261156559, "rewards/accuracy_reward/std": 0.26274051517248154, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 2888 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 843.8393249511719, "completions/mean_terminated_length": 722.4425811767578, "completions/min_length": 207.25, "completions/min_terminated_length": 207.25, "epoch": 0.8629676648495258, "grad_norm": 0.463684618473053, "kl": 2.107421875, "learning_rate": 5.735850890782158e-07, "loss": 0.1097, "num_tokens": 1380647901.0, "reward": 0.6607142984867096, "reward_std": 0.11195475980639458, "rewards/accuracy_reward/mean": 0.1696428577415645, "rewards/accuracy_reward/std": 0.3352780118584633, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491071417927742, "rewards/tag_count_reward/std": 0.044901167973876, "step": 2889 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 869.3772735595703, "completions/mean_terminated_length": 760.5159454345703, "completions/min_length": 390.5, "completions/min_terminated_length": 390.5, "epoch": 0.8632663729370472, "grad_norm": 0.1997038722038269, "kl": 2.3984375, "learning_rate": 5.69949430126333e-07, "loss": 0.1302, "num_tokens": 1381111254.0, "reward": 0.6294643208384514, "reward_std": 0.13519529346376657, "rewards/accuracy_reward/mean": 0.14062500186264515, "rewards/accuracy_reward/std": 0.27991194278001785, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.050577715039253235, "step": 2890 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 878.1830749511719, "completions/mean_terminated_length": 736.7154846191406, "completions/min_length": 358.5, "completions/min_terminated_length": 358.5, "epoch": 0.8635650810245687, "grad_norm": 0.2726598381996155, "kl": 1.845703125, "learning_rate": 5.663249921017477e-07, "loss": 0.0797, "num_tokens": 1381575656.0, "reward": 0.6925223469734192, "reward_std": 0.1656021699309349, "rewards/accuracy_reward/mean": 0.2064732126891613, "rewards/accuracy_reward/std": 0.39700889587402344, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 2891 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.33705357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 821.5089569091797, "completions/mean_terminated_length": 718.8978271484375, "completions/min_length": 302.5, "completions/min_terminated_length": 302.5, "epoch": 0.8638637891120902, "grad_norm": 0.3211408257484436, "kl": 2.220703125, "learning_rate": 5.627117793172221e-07, "loss": 0.1305, "num_tokens": 1382015788.0, "reward": 0.6612723469734192, "reward_std": 0.12519781244918704, "rewards/accuracy_reward/mean": 0.1696428619325161, "rewards/accuracy_reward/std": 0.3055279850959778, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 2892 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 800.7321624755859, "completions/mean_terminated_length": 683.7216033935547, "completions/min_length": 281.25, "completions/min_terminated_length": 281.25, "epoch": 0.8641624971996117, "grad_norm": 0.32268238067626953, "kl": 2.513671875, "learning_rate": 5.591097960721581e-07, "loss": 0.1512, "num_tokens": 1382444708.0, "reward": 0.663504496216774, "reward_std": 0.16679386049509048, "rewards/accuracy_reward/mean": 0.17410714365541935, "rewards/accuracy_reward/std": 0.3523515537381172, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.047861308325082064, "step": 2893 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 886.0312957763672, "completions/mean_terminated_length": 775.6980438232422, "completions/min_length": 367.75, "completions/min_terminated_length": 367.75, "epoch": 0.8644612052871331, "grad_norm": 0.31072142720222473, "kl": 1.552734375, "learning_rate": 5.555190466525984e-07, "loss": 0.0774, "num_tokens": 1382916626.0, "reward": 0.6997768133878708, "reward_std": 0.2000470757484436, "rewards/accuracy_reward/mean": 0.2053571417927742, "rewards/accuracy_reward/std": 0.4012892246246338, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03549952572211623, "step": 2894 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31473214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.25, "completions/mean_length": 819.404052734375, "completions/mean_terminated_length": 727.6121063232422, "completions/min_length": 307.0, "completions/min_terminated_length": 307.0, "epoch": 0.8647599133746546, "grad_norm": 0.40461814403533936, "kl": 1.43310546875, "learning_rate": 5.519395353312195e-07, "loss": 0.0768, "num_tokens": 1383349495.0, "reward": 0.75167416036129, "reward_std": 0.23488258570432663, "rewards/accuracy_reward/mean": 0.2566964291036129, "rewards/accuracy_reward/std": 0.43062080442905426, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.030409175902605057, "step": 2895 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 872.9107513427734, "completions/mean_terminated_length": 774.9937744140625, "completions/min_length": 387.75, "completions/min_terminated_length": 387.75, "epoch": 0.865058621462176, "grad_norm": 0.35971933603286743, "kl": 2.2275390625, "learning_rate": 5.483712663673224e-07, "loss": 0.1134, "num_tokens": 1383815023.0, "reward": 0.6233259290456772, "reward_std": 0.14180130884051323, "rewards/accuracy_reward/mean": 0.13169642398133874, "rewards/accuracy_reward/std": 0.26562497206032276, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.042382154148072004, "step": 2896 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35044642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.5, "completions/mean_length": 843.825927734375, "completions/mean_terminated_length": 747.1349945068359, "completions/min_length": 401.75, "completions/min_terminated_length": 401.75, "epoch": 0.8653573295496976, "grad_norm": 0.5289591550827026, "kl": 2.029296875, "learning_rate": 5.448142440068316e-07, "loss": 0.116, "num_tokens": 1384272929.0, "reward": 0.6328125298023224, "reward_std": 0.14124542102217674, "rewards/accuracy_reward/mean": 0.14062500232830644, "rewards/accuracy_reward/std": 0.3225843720138073, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0410674219019711, "step": 2897 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 861.8861999511719, "completions/mean_terminated_length": 745.6038665771484, "completions/min_length": 395.75, "completions/min_terminated_length": 395.75, "epoch": 0.865656037637219, "grad_norm": 0.38346928358078003, "kl": 2.89453125, "learning_rate": 5.412684724822914e-07, "loss": 0.1471, "num_tokens": 1384730366.0, "reward": 0.5675223544239998, "reward_std": 0.13730175327509642, "rewards/accuracy_reward/mean": 0.08035714272409678, "rewards/accuracy_reward/std": 0.22099021077156067, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05446719843894243, "step": 2898 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.29910714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 810.6718902587891, "completions/mean_terminated_length": 721.8199768066406, "completions/min_length": 315.0, "completions/min_terminated_length": 315.0, "epoch": 0.8659547457247405, "grad_norm": 0.25135892629623413, "kl": 2.29296875, "learning_rate": 5.377339560128536e-07, "loss": 0.1328, "num_tokens": 1385171995.0, "reward": 0.7265625447034836, "reward_std": 0.2057291530072689, "rewards/accuracy_reward/mean": 0.2433035708963871, "rewards/accuracy_reward/std": 0.42534323781728745, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 2899 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 834.2053833007812, "completions/mean_terminated_length": 734.4921722412109, "completions/min_length": 329.5, "completions/min_terminated_length": 329.5, "epoch": 0.8662534538122619, "grad_norm": 0.4658731520175934, "kl": 2.07421875, "learning_rate": 5.342106988042839e-07, "loss": 0.117, "num_tokens": 1385619607.0, "reward": 0.7421875298023224, "reward_std": 0.22507184743881226, "rewards/accuracy_reward/mean": 0.2500000011641532, "rewards/accuracy_reward/std": 0.3687733765691519, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.042172474320977926, "step": 2900 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3191964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 838.5670013427734, "completions/mean_terminated_length": 754.1256561279297, "completions/min_length": 393.5, "completions/min_terminated_length": 393.5, "epoch": 0.8665521618997835, "grad_norm": 0.2755109667778015, "kl": 1.359375, "learning_rate": 5.306987050489442e-07, "loss": 0.0793, "num_tokens": 1386065861.0, "reward": 0.698660746216774, "reward_std": 0.17948300507850945, "rewards/accuracy_reward/mean": 0.215773805975914, "rewards/accuracy_reward/std": 0.3388148844242096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.495535708963871, "rewards/tag_count_reward/std": 0.03267050301656127, "step": 2901 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 867.7232513427734, "completions/mean_terminated_length": 742.2543792724609, "completions/min_length": 223.75, "completions/min_terminated_length": 223.75, "epoch": 0.8668508699873049, "grad_norm": 0.3049336373806, "kl": 1.5458984375, "learning_rate": 5.271979789257986e-07, "loss": 0.0843, "num_tokens": 1386528377.0, "reward": 0.7126116305589676, "reward_std": 0.19591986387968063, "rewards/accuracy_reward/mean": 0.21874999906867743, "rewards/accuracy_reward/std": 0.35586006194353104, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03782916208729148, "step": 2902 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.75, "completions/mean_length": 856.6786041259766, "completions/mean_terminated_length": 751.2446746826172, "completions/min_length": 301.0, "completions/min_terminated_length": 301.0, "epoch": 0.8671495780748264, "grad_norm": 0.367427259683609, "kl": 2.236328125, "learning_rate": 5.237085246004015e-07, "loss": 0.1304, "num_tokens": 1386985753.0, "reward": 0.671316996216774, "reward_std": 0.15824388340115547, "rewards/accuracy_reward/mean": 0.18080356903374195, "rewards/accuracy_reward/std": 0.38031821697950363, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 2903 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 859.7053833007812, "completions/mean_terminated_length": 760.3754730224609, "completions/min_length": 338.25, "completions/min_terminated_length": 338.25, "epoch": 0.8674482861623478, "grad_norm": 0.26808616518974304, "kl": 2.107421875, "learning_rate": 5.20230346224897e-07, "loss": 0.1019, "num_tokens": 1387447749.0, "reward": 0.6032366156578064, "reward_std": 0.09132527932524681, "rewards/accuracy_reward/mean": 0.11160714109428227, "rewards/accuracy_reward/std": 0.28659105114638805, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098951727152, "step": 2904 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 843.9219207763672, "completions/mean_terminated_length": 733.8138427734375, "completions/min_length": 303.5, "completions/min_terminated_length": 303.5, "epoch": 0.8677469942498693, "grad_norm": 0.4142961800098419, "kl": 2.5703125, "learning_rate": 5.167634479380068e-07, "loss": 0.1353, "num_tokens": 1387907938.0, "reward": 0.5987723469734192, "reward_std": 0.16850651241838932, "rewards/accuracy_reward/mean": 0.10937499906867743, "rewards/accuracy_reward/std": 0.30232106149196625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04960631299763918, "step": 2905 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34821428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 820.1652069091797, "completions/mean_terminated_length": 720.4815673828125, "completions/min_length": 286.75, "completions/min_terminated_length": 286.75, "epoch": 0.8680457023373908, "grad_norm": 0.5076075196266174, "kl": 1.6640625, "learning_rate": 5.133078338650376e-07, "loss": 0.1016, "num_tokens": 1388360172.0, "reward": 0.6992187798023224, "reward_std": 0.1566807497292757, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.39055371284484863, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 2906 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 864.1875457763672, "completions/mean_terminated_length": 764.0659027099609, "completions/min_length": 391.0, "completions/min_terminated_length": 391.0, "epoch": 0.8683444104249123, "grad_norm": 0.23460887372493744, "kl": 2.439453125, "learning_rate": 5.098635081178615e-07, "loss": 0.1229, "num_tokens": 1388822464.0, "reward": 0.5876116454601288, "reward_std": 0.07495741359889507, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.2882710173726082, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2907 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1023.25, "completions/mean_length": 896.8437805175781, "completions/mean_terminated_length": 774.3844451904297, "completions/min_length": 283.25, "completions/min_terminated_length": 283.25, "epoch": 0.8686431185124337, "grad_norm": 0.3532984256744385, "kl": 2.8515625, "learning_rate": 5.064304747949233e-07, "loss": 0.1432, "num_tokens": 1389297482.0, "reward": 0.5809151977300644, "reward_std": 0.14464334398508072, "rewards/accuracy_reward/mean": 0.09598214458674192, "rewards/accuracy_reward/std": 0.2882591299712658, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484933041036129, "rewards/tag_count_reward/std": 0.05737343430519104, "step": 2908 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 878.0245971679688, "completions/mean_terminated_length": 789.8108673095703, "completions/min_length": 435.75, "completions/min_terminated_length": 435.75, "epoch": 0.8689418265999552, "grad_norm": 0.2803058624267578, "kl": 1.7392578125, "learning_rate": 5.030087379812299e-07, "loss": 0.0866, "num_tokens": 1389763493.0, "reward": 0.6093750149011612, "reward_std": 0.16543840616941452, "rewards/accuracy_reward/mean": 0.11607142491266131, "rewards/accuracy_reward/std": 0.28842346742749214, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.0371446181088686, "step": 2909 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 874.3906707763672, "completions/mean_terminated_length": 762.4765472412109, "completions/min_length": 322.0, "completions/min_terminated_length": 322.0, "epoch": 0.8692405346874766, "grad_norm": 0.21797850728034973, "kl": 2.380859375, "learning_rate": 4.995983017483463e-07, "loss": 0.1226, "num_tokens": 1390235300.0, "reward": 0.6266741454601288, "reward_std": 0.13927063532173634, "rewards/accuracy_reward/mean": 0.13616071734577417, "rewards/accuracy_reward/std": 0.3269207179546356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04706668108701706, "step": 2910 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 887.5357513427734, "completions/mean_terminated_length": 767.7803192138672, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "epoch": 0.8695392427749982, "grad_norm": 0.22286850214004517, "kl": 1.766357421875, "learning_rate": 4.961991701543889e-07, "loss": 0.0932, "num_tokens": 1390703636.0, "reward": 0.565848246216774, "reward_std": 0.10890429187566042, "rewards/accuracy_reward/mean": 0.0736607147846371, "rewards/accuracy_reward/std": 0.23004659079015255, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.03658695984631777, "step": 2911 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 865.7768402099609, "completions/mean_terminated_length": 747.7375183105469, "completions/min_length": 333.25, "completions/min_terminated_length": 333.25, "epoch": 0.8698379508625196, "grad_norm": 0.20322935283184052, "kl": 2.369140625, "learning_rate": 4.928113472440255e-07, "loss": 0.1144, "num_tokens": 1391166304.0, "reward": 0.7059152126312256, "reward_std": 0.1471653040498495, "rewards/accuracy_reward/mean": 0.2165178544819355, "rewards/accuracy_reward/std": 0.41172393411397934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 2912 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 841.7545013427734, "completions/mean_terminated_length": 733.9255218505859, "completions/min_length": 307.75, "completions/min_terminated_length": 307.75, "epoch": 0.8701366589500411, "grad_norm": 0.17232222855091095, "kl": 1.7548828125, "learning_rate": 4.894348370484648e-07, "loss": 0.0834, "num_tokens": 1391622274.0, "reward": 0.6824777126312256, "reward_std": 0.12730063498020172, "rewards/accuracy_reward/mean": 0.1897321413271129, "rewards/accuracy_reward/std": 0.35696468502283096, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 2913 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39062499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 870.6607513427734, "completions/mean_terminated_length": 776.8827362060547, "completions/min_length": 393.25, "completions/min_terminated_length": 393.25, "epoch": 0.8704353670375625, "grad_norm": 0.3866705894470215, "kl": 2.8359375, "learning_rate": 4.860696435854573e-07, "loss": 0.1501, "num_tokens": 1392089482.0, "reward": 0.6333705633878708, "reward_std": 0.2060442790389061, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.3536277636885643, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.05541300866752863, "step": 2914 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35044642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.25, "completions/mean_length": 839.5089721679688, "completions/mean_terminated_length": 740.3452911376953, "completions/min_length": 382.75, "completions/min_terminated_length": 382.75, "epoch": 0.870734075125084, "grad_norm": 0.21042457222938538, "kl": 1.8671875, "learning_rate": 4.827157708592834e-07, "loss": 0.0944, "num_tokens": 1392527518.0, "reward": 0.7449776828289032, "reward_std": 0.1733943484723568, "rewards/accuracy_reward/mean": 0.25223213620483875, "rewards/accuracy_reward/std": 0.39767856895923615, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2915 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 820.6428833007812, "completions/mean_terminated_length": 706.2695922851562, "completions/min_length": 326.75, "completions/min_terminated_length": 326.75, "epoch": 0.8710327832126055, "grad_norm": 0.6700782775878906, "kl": 2.98046875, "learning_rate": 4.793732228607573e-07, "loss": 0.1522, "num_tokens": 1392963678.0, "reward": 0.6462053805589676, "reward_std": 0.17835531756281853, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.36036891490221024, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 2916 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 857.7745971679688, "completions/mean_terminated_length": 760.7347717285156, "completions/min_length": 445.0, "completions/min_terminated_length": 445.0, "epoch": 0.871331491300127, "grad_norm": 0.340165913105011, "kl": 3.052734375, "learning_rate": 4.7604200356721644e-07, "loss": 0.1601, "num_tokens": 1393423529.0, "reward": 0.7126116454601288, "reward_std": 0.2001859676092863, "rewards/accuracy_reward/mean": 0.22767857182770967, "rewards/accuracy_reward/std": 0.3781936392188072, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05936532001942396, "step": 2917 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 871.5625610351562, "completions/mean_terminated_length": 763.7730712890625, "completions/min_length": 304.25, "completions/min_terminated_length": 304.25, "epoch": 0.8716301993876484, "grad_norm": 0.5038044452667236, "kl": 1.5009765625, "learning_rate": 4.727221169425178e-07, "loss": 0.0796, "num_tokens": 1393890325.0, "reward": 0.5842634215950966, "reward_std": 0.10902335587888956, "rewards/accuracy_reward/mean": 0.09151785774156451, "rewards/accuracy_reward/std": 0.2226949781179428, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 2918 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 882.529052734375, "completions/mean_terminated_length": 787.3441467285156, "completions/min_length": 368.25, "completions/min_terminated_length": 368.25, "epoch": 0.8719289074751699, "grad_norm": 0.23252548277378082, "kl": 2.8984375, "learning_rate": 4.6941356693703034e-07, "loss": 0.1491, "num_tokens": 1394364642.0, "reward": 0.5842634290456772, "reward_std": 0.13391990214586258, "rewards/accuracy_reward/mean": 0.09821428637951612, "rewards/accuracy_reward/std": 0.2840290293097496, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05749546363949776, "step": 2919 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5959821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 946.794677734375, "completions/mean_terminated_length": 838.0462951660156, "completions/min_length": 533.5, "completions/min_terminated_length": 533.5, "epoch": 0.8722276155626913, "grad_norm": 0.3532910943031311, "kl": 2.9609375, "learning_rate": 4.6611635748763926e-07, "loss": 0.1366, "num_tokens": 1394865910.0, "reward": 0.6093750298023224, "reward_std": 0.14677446521818638, "rewards/accuracy_reward/mean": 0.12276785913854837, "rewards/accuracy_reward/std": 0.3157297447323799, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05563815962523222, "step": 2920 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 816.9129943847656, "completions/mean_terminated_length": 720.3993225097656, "completions/min_length": 314.25, "completions/min_terminated_length": 314.25, "epoch": 0.8725263236502129, "grad_norm": 0.22863949835300446, "kl": 2.55859375, "learning_rate": 4.628304925177318e-07, "loss": 0.1289, "num_tokens": 1395309375.0, "reward": 0.6635044813156128, "reward_std": 0.1769836936146021, "rewards/accuracy_reward/mean": 0.1770833316259086, "rewards/accuracy_reward/std": 0.33422696962952614, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05405060388147831, "step": 2921 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 872.5179138183594, "completions/mean_terminated_length": 751.4740295410156, "completions/min_length": 324.5, "completions/min_terminated_length": 324.5, "epoch": 0.8728250317377343, "grad_norm": 0.5214491486549377, "kl": 2.05078125, "learning_rate": 4.5955597593719593e-07, "loss": 0.1062, "num_tokens": 1395772823.0, "reward": 0.5881696790456772, "reward_std": 0.14369411766529083, "rewards/accuracy_reward/mean": 0.09598214272409678, "rewards/accuracy_reward/std": 0.290914922952652, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 2922 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 812.544677734375, "completions/mean_terminated_length": 700.9851226806641, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "epoch": 0.8731237398252558, "grad_norm": 0.3116191029548645, "kl": 2.447265625, "learning_rate": 4.562928116424181e-07, "loss": 0.1336, "num_tokens": 1396209979.0, "reward": 0.655691996216774, "reward_std": 0.1553240716457367, "rewards/accuracy_reward/mean": 0.1651785746216774, "rewards/accuracy_reward/std": 0.3565993309020996, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241139769554, "step": 2923 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.28794642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 835.4375305175781, "completions/mean_terminated_length": 755.7158660888672, "completions/min_length": 418.5, "completions/min_terminated_length": 418.5, "epoch": 0.8734224479127772, "grad_norm": 0.32324427366256714, "kl": 1.3994140625, "learning_rate": 4.530410035162769e-07, "loss": 0.0796, "num_tokens": 1396656111.0, "reward": 0.7165178805589676, "reward_std": 0.1479460783302784, "rewards/accuracy_reward/mean": 0.22098214738070965, "rewards/accuracy_reward/std": 0.39427004754543304, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.02769277011975646, "step": 2924 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 826.513427734375, "completions/mean_terminated_length": 728.0224914550781, "completions/min_length": 334.0, "completions/min_terminated_length": 334.0, "epoch": 0.8737211560002988, "grad_norm": 0.17158013582229614, "kl": 2.041015625, "learning_rate": 4.498005554281337e-07, "loss": 0.1031, "num_tokens": 1397099285.0, "reward": 0.6735491305589676, "reward_std": 0.1465628705918789, "rewards/accuracy_reward/mean": 0.1830357159487903, "rewards/accuracy_reward/std": 0.3461626395583153, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04517605667933822, "step": 2925 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 854.013427734375, "completions/mean_terminated_length": 760.2974395751953, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "epoch": 0.8740198640878202, "grad_norm": 0.3023022413253784, "kl": 1.740234375, "learning_rate": 4.465714712338398e-07, "loss": 0.1002, "num_tokens": 1397555355.0, "reward": 0.7031250298023224, "reward_std": 0.19016292691230774, "rewards/accuracy_reward/mean": 0.20982142351567745, "rewards/accuracy_reward/std": 0.39360252022743225, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2926 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4776785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 882.6428985595703, "completions/mean_terminated_length": 758.8761291503906, "completions/min_length": 385.5, "completions/min_terminated_length": 385.5, "epoch": 0.8743185721753417, "grad_norm": 0.2411377727985382, "kl": 2.34375, "learning_rate": 4.4335375477571497e-07, "loss": 0.1098, "num_tokens": 1398020155.0, "reward": 0.6863839402794838, "reward_std": 0.11882856860756874, "rewards/accuracy_reward/mean": 0.19642857206054032, "rewards/accuracy_reward/std": 0.34507819078862667, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046273752581328154, "step": 2927 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35714285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 837.9553985595703, "completions/mean_terminated_length": 733.1730499267578, "completions/min_length": 386.5, "completions/min_terminated_length": 386.5, "epoch": 0.8746172802628631, "grad_norm": 0.3331218361854553, "kl": 2.8203125, "learning_rate": 4.401474098825631e-07, "loss": 0.1572, "num_tokens": 1398470231.0, "reward": 0.8186384290456772, "reward_std": 0.16810750402510166, "rewards/accuracy_reward/mean": 0.3303571417927742, "rewards/accuracy_reward/std": 0.4651147425174713, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05277834925800562, "step": 2928 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32589285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 789.0156707763672, "completions/mean_terminated_length": 676.6045837402344, "completions/min_length": 297.75, "completions/min_terminated_length": 297.75, "epoch": 0.8749159883503845, "grad_norm": 0.34847116470336914, "kl": 2.630859375, "learning_rate": 4.3695244036964567e-07, "loss": 0.1693, "num_tokens": 1398896942.0, "reward": 0.7315848469734192, "reward_std": 0.1553210155107081, "rewards/accuracy_reward/mean": 0.2492559514939785, "rewards/accuracy_reward/std": 0.41376588493585587, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05133028235286474, "step": 2929 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 877.0848693847656, "completions/mean_terminated_length": 778.8320159912109, "completions/min_length": 318.25, "completions/min_terminated_length": 318.25, "epoch": 0.8752146964379061, "grad_norm": 0.2720431089401245, "kl": 1.533203125, "learning_rate": 4.337688500386983e-07, "loss": 0.0759, "num_tokens": 1399372260.0, "reward": 0.6210937947034836, "reward_std": 0.13976527098566294, "rewards/accuracy_reward/mean": 0.1272321401629597, "rewards/accuracy_reward/std": 0.2805972807109356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 2930 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 869.1897583007812, "completions/mean_terminated_length": 754.4288787841797, "completions/min_length": 216.25, "completions/min_terminated_length": 216.25, "epoch": 0.8755134045254275, "grad_norm": 0.3951379358768463, "kl": 2.521484375, "learning_rate": 4.305966426779118e-07, "loss": 0.1276, "num_tokens": 1399840857.0, "reward": 0.628348246216774, "reward_std": 0.15782175958156586, "rewards/accuracy_reward/mean": 0.13839285587891936, "rewards/accuracy_reward/std": 0.3204500153660774, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04558529471978545, "step": 2931 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 840.5111999511719, "completions/mean_terminated_length": 741.0862579345703, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "epoch": 0.875812112612949, "grad_norm": 0.2990437150001526, "kl": 2.439453125, "learning_rate": 4.2743582206193124e-07, "loss": 0.1434, "num_tokens": 1400286654.0, "reward": 0.7154018059372902, "reward_std": 0.16990202385932207, "rewards/accuracy_reward/mean": 0.2254464253783226, "rewards/accuracy_reward/std": 0.338262215256691, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04702208936214447, "step": 2932 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3973214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 831.4911041259766, "completions/mean_terminated_length": 706.9509735107422, "completions/min_length": 232.25, "completions/min_terminated_length": 232.25, "epoch": 0.8761108207004704, "grad_norm": 0.3018784821033478, "kl": 1.998046875, "learning_rate": 4.2428639195185585e-07, "loss": 0.105, "num_tokens": 1400739866.0, "reward": 0.6462053954601288, "reward_std": 0.17222376400604844, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.2910507470369339, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 2933 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39508928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 861.7120819091797, "completions/mean_terminated_length": 766.7011413574219, "completions/min_length": 323.75, "completions/min_terminated_length": 323.75, "epoch": 0.8764095287879919, "grad_norm": 0.18692541122436523, "kl": 1.703125, "learning_rate": 4.2114835609522784e-07, "loss": 0.0872, "num_tokens": 1401192905.0, "reward": 0.6177455633878708, "reward_std": 0.1190662500448525, "rewards/accuracy_reward/mean": 0.1249999962747097, "rewards/accuracy_reward/std": 0.280732236802578, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.0412445142865181, "step": 2934 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3616071428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 866.6361999511719, "completions/mean_terminated_length": 777.9381408691406, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "epoch": 0.8767082368755134, "grad_norm": 0.22951313853263855, "kl": 1.4921875, "learning_rate": 4.180217182260338e-07, "loss": 0.0727, "num_tokens": 1401648502.0, "reward": 0.6004464626312256, "reward_std": 0.10184746980667114, "rewards/accuracy_reward/mean": 0.10714285564608872, "rewards/accuracy_reward/std": 0.2745047677308321, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2935 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 877.1317291259766, "completions/mean_terminated_length": 746.8416748046875, "completions/min_length": 292.75, "completions/min_terminated_length": 292.75, "epoch": 0.8770069449630349, "grad_norm": 0.24938461184501648, "kl": 2.232421875, "learning_rate": 4.149064820646953e-07, "loss": 0.1136, "num_tokens": 1402110385.0, "reward": 0.640066996216774, "reward_std": 0.1407684162259102, "rewards/accuracy_reward/mean": 0.1517857201397419, "rewards/accuracy_reward/std": 0.2002772092819214, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05243501905351877, "step": 2936 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 858.1295013427734, "completions/mean_terminated_length": 755.4784545898438, "completions/min_length": 299.5, "completions/min_terminated_length": 299.5, "epoch": 0.8773056530505563, "grad_norm": 0.5085119605064392, "kl": 2.03515625, "learning_rate": 4.118026513180695e-07, "loss": 0.1009, "num_tokens": 1402570891.0, "reward": 0.5870535969734192, "reward_std": 0.15273034572601318, "rewards/accuracy_reward/mean": 0.09598214132711291, "rewards/accuracy_reward/std": 0.2768496051430702, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.0447555473074317, "step": 2937 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 854.1629943847656, "completions/mean_terminated_length": 746.9849853515625, "completions/min_length": 397.75, "completions/min_terminated_length": 397.75, "epoch": 0.8776043611380778, "grad_norm": 0.2482069581747055, "kl": 2.291015625, "learning_rate": 4.0871022967943985e-07, "loss": 0.1251, "num_tokens": 1403024100.0, "reward": 0.591517873108387, "reward_std": 0.11668264400213957, "rewards/accuracy_reward/mean": 0.10044642724096775, "rewards/accuracy_reward/std": 0.2521771490573883, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 2938 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 877.0491333007812, "completions/mean_terminated_length": 780.4354705810547, "completions/min_length": 413.0, "completions/min_terminated_length": 413.0, "epoch": 0.8779030692255992, "grad_norm": 0.2677750289440155, "kl": 2.0390625, "learning_rate": 4.056292208285162e-07, "loss": 0.1219, "num_tokens": 1403487114.0, "reward": 0.6294643133878708, "reward_std": 0.15127426199615002, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.33124281093478203, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 2939 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 873.8527221679688, "completions/mean_terminated_length": 759.1443176269531, "completions/min_length": 339.75, "completions/min_terminated_length": 339.75, "epoch": 0.8782017773131208, "grad_norm": 0.7199548482894897, "kl": 2.48046875, "learning_rate": 4.025596284314259e-07, "loss": 0.1314, "num_tokens": 1403958024.0, "reward": 0.6227678880095482, "reward_std": 0.12283437978476286, "rewards/accuracy_reward/mean": 0.1339285708963871, "rewards/accuracy_reward/std": 0.2866944298148155, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 2940 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.25, "completions/mean_length": 853.7053985595703, "completions/mean_terminated_length": 739.7244415283203, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "epoch": 0.8785004854006422, "grad_norm": 0.3059815764427185, "kl": 3.17578125, "learning_rate": 3.99501456140714e-07, "loss": 0.1816, "num_tokens": 1404407732.0, "reward": 0.6947545111179352, "reward_std": 0.17011429369449615, "rewards/accuracy_reward/mean": 0.2180059514939785, "rewards/accuracy_reward/std": 0.37452471628785133, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330261349678, "rewards/tag_count_reward/std": 0.058406153693795204, "step": 2941 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.0, "completions/mean_length": 834.3460235595703, "completions/mean_terminated_length": 742.8207702636719, "completions/min_length": 241.25, "completions/min_terminated_length": 241.25, "epoch": 0.8787991934881637, "grad_norm": 0.4809933006763458, "kl": 1.92578125, "learning_rate": 3.964547075953329e-07, "loss": 0.1148, "num_tokens": 1404861855.0, "reward": 0.7070312798023224, "reward_std": 0.08832294028252363, "rewards/accuracy_reward/mean": 0.2142857164144516, "rewards/accuracy_reward/std": 0.3911965414881706, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 2942 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44642857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 893.0156707763672, "completions/mean_terminated_length": 788.7942962646484, "completions/min_length": 453.75, "completions/min_terminated_length": 453.75, "epoch": 0.8790979015756851, "grad_norm": 0.2782459259033203, "kl": 2.25, "learning_rate": 3.9341938642064814e-07, "loss": 0.1265, "num_tokens": 1405331366.0, "reward": 0.637276828289032, "reward_std": 0.1521524116396904, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.33707452937960625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04903263598680496, "step": 2943 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 841.1183471679688, "completions/mean_terminated_length": 732.6043853759766, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "epoch": 0.8793966096632067, "grad_norm": 0.31322580575942993, "kl": 2.720703125, "learning_rate": 3.9039549622841844e-07, "loss": 0.142, "num_tokens": 1405779323.0, "reward": 0.678013414144516, "reward_std": 0.13647200725972652, "rewards/accuracy_reward/mean": 0.19940476235933602, "rewards/accuracy_reward/std": 0.34099997393786907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05243533570319414, "step": 2944 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 852.8326263427734, "completions/mean_terminated_length": 728.2427520751953, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "epoch": 0.8796953177507281, "grad_norm": 0.41580280661582947, "kl": 2.2216796875, "learning_rate": 3.8738304061681107e-07, "loss": 0.1108, "num_tokens": 1406231488.0, "reward": 0.6372768133878708, "reward_std": 0.15095243696123362, "rewards/accuracy_reward/mean": 0.14732142724096775, "rewards/accuracy_reward/std": 0.34880251437425613, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04195561446249485, "step": 2945 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 894.1830749511719, "completions/mean_terminated_length": 783.6611328125, "completions/min_length": 415.75, "completions/min_terminated_length": 415.75, "epoch": 0.8799940258382496, "grad_norm": 0.2802397906780243, "kl": 2.111328125, "learning_rate": 3.8438202317037987e-07, "loss": 0.1008, "num_tokens": 1406718098.0, "reward": 0.5954241305589676, "reward_std": 0.130304753780365, "rewards/accuracy_reward/mean": 0.10491071548312902, "rewards/accuracy_reward/std": 0.2976754680275917, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241046637297, "step": 2946 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 866.6295013427734, "completions/mean_terminated_length": 767.2937927246094, "completions/min_length": 353.0, "completions/min_terminated_length": 353.0, "epoch": 0.880292733925771, "grad_norm": 0.32936060428619385, "kl": 1.7099609375, "learning_rate": 3.8139244746007276e-07, "loss": 0.0758, "num_tokens": 1407180444.0, "reward": 0.722098246216774, "reward_std": 0.15788634307682514, "rewards/accuracy_reward/mean": 0.22991071385331452, "rewards/accuracy_reward/std": 0.35913957469165325, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04132169345393777, "step": 2947 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 854.7455902099609, "completions/mean_terminated_length": 735.9359588623047, "completions/min_length": 282.0, "completions/min_terminated_length": 282.0, "epoch": 0.8805914420132925, "grad_norm": 0.3124113082885742, "kl": 1.6982421875, "learning_rate": 3.784143170432164e-07, "loss": 0.0918, "num_tokens": 1407632938.0, "reward": 0.6361607313156128, "reward_std": 0.14962408691644669, "rewards/accuracy_reward/mean": 0.14285714644938707, "rewards/accuracy_reward/std": 0.3319980949163437, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 2948 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 825.0580749511719, "completions/mean_terminated_length": 709.4481658935547, "completions/min_length": 310.75, "completions/min_terminated_length": 310.75, "epoch": 0.880890150100814, "grad_norm": 0.25998902320861816, "kl": 2.4453125, "learning_rate": 3.7544763546352834e-07, "loss": 0.1362, "num_tokens": 1408069316.0, "reward": 0.6573660969734192, "reward_std": 0.14700570702552795, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3654145449399948, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04923219606280327, "step": 2949 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 859.6361999511719, "completions/mean_terminated_length": 755.3239593505859, "completions/min_length": 360.75, "completions/min_terminated_length": 360.75, "epoch": 0.8811888581883355, "grad_norm": 0.2949026823043823, "kl": 2.35546875, "learning_rate": 3.724924062510926e-07, "loss": 0.1209, "num_tokens": 1408534001.0, "reward": 0.6060268133878708, "reward_std": 0.14538298547267914, "rewards/accuracy_reward/mean": 0.11793154571205378, "rewards/accuracy_reward/std": 0.3149336166679859, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04863459337502718, "step": 2950 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 833.3058319091797, "completions/mean_terminated_length": 706.6571197509766, "completions/min_length": 348.75, "completions/min_terminated_length": 348.75, "epoch": 0.8814875662758569, "grad_norm": 0.31211888790130615, "kl": 2.1953125, "learning_rate": 3.6954863292237297e-07, "loss": 0.1151, "num_tokens": 1408990346.0, "reward": 0.5206473469734192, "reward_std": 0.09626738587394357, "rewards/accuracy_reward/mean": 0.0290178582072258, "rewards/accuracy_reward/std": 0.13847661390900612, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 2951 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 866.7745971679688, "completions/mean_terminated_length": 758.1355895996094, "completions/min_length": 438.5, "completions/min_terminated_length": 438.5, "epoch": 0.8817862743633784, "grad_norm": 0.244811549782753, "kl": 2.767578125, "learning_rate": 3.666163189802008e-07, "loss": 0.1382, "num_tokens": 1409454165.0, "reward": 0.6808035969734192, "reward_std": 0.18859289214015007, "rewards/accuracy_reward/mean": 0.1941964291036129, "rewards/accuracy_reward/std": 0.3855440691113472, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05536533612757921, "step": 2952 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44419642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 871.232177734375, "completions/mean_terminated_length": 762.8475646972656, "completions/min_length": 376.25, "completions/min_terminated_length": 376.25, "epoch": 0.8820849824508998, "grad_norm": 0.2129395455121994, "kl": 2.380859375, "learning_rate": 3.6369546791377054e-07, "loss": 0.1252, "num_tokens": 1409916733.0, "reward": 0.7594866305589676, "reward_std": 0.20916515588760376, "rewards/accuracy_reward/mean": 0.2700892873108387, "rewards/accuracy_reward/std": 0.43596646934747696, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810296744108, "step": 2953 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 864.9531707763672, "completions/mean_terminated_length": 762.5835418701172, "completions/min_length": 337.25, "completions/min_terminated_length": 337.25, "epoch": 0.8823836905384214, "grad_norm": 0.24210159480571747, "kl": 2.04296875, "learning_rate": 3.607860831986354e-07, "loss": 0.1126, "num_tokens": 1410370520.0, "reward": 0.6238839626312256, "reward_std": 0.16512721870094538, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.2859034091234207, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.03613344579935074, "step": 2954 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 844.6339721679688, "completions/mean_terminated_length": 722.2577362060547, "completions/min_length": 293.0, "completions/min_terminated_length": 293.0, "epoch": 0.8826823986259428, "grad_norm": 0.4619353115558624, "kl": 2.484375, "learning_rate": 3.5788816829670723e-07, "loss": 0.1322, "num_tokens": 1410819012.0, "reward": 0.6551339477300644, "reward_std": 0.23300255462527275, "rewards/accuracy_reward/mean": 0.16517856903374195, "rewards/accuracy_reward/std": 0.37063802033662796, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.048545535653829575, "step": 2955 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 851.5580749511719, "completions/mean_terminated_length": 741.6747741699219, "completions/min_length": 264.75, "completions/min_terminated_length": 264.75, "epoch": 0.8829811067134643, "grad_norm": 0.2840491235256195, "kl": 2.78125, "learning_rate": 3.550017266562489e-07, "loss": 0.1345, "num_tokens": 1411273278.0, "reward": 0.5876116454601288, "reward_std": 0.12613153457641602, "rewards/accuracy_reward/mean": 0.10156249813735485, "rewards/accuracy_reward/std": 0.29274191707372665, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05464820470660925, "step": 2956 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 823.7098541259766, "completions/mean_terminated_length": 720.285400390625, "completions/min_length": 341.5, "completions/min_terminated_length": 341.5, "epoch": 0.8832798148009857, "grad_norm": 0.17489461600780487, "kl": 1.380859375, "learning_rate": 3.5212676171187065e-07, "loss": 0.0635, "num_tokens": 1411710716.0, "reward": 0.7008928805589676, "reward_std": 0.09250064985826612, "rewards/accuracy_reward/mean": 0.2075892835855484, "rewards/accuracy_reward/std": 0.3368992581963539, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03992978110909462, "step": 2957 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 901.0937957763672, "completions/mean_terminated_length": 804.7708129882812, "completions/min_length": 320.5, "completions/min_terminated_length": 320.5, "epoch": 0.8835785228885072, "grad_norm": 0.27884721755981445, "kl": 1.75390625, "learning_rate": 3.492632768845261e-07, "loss": 0.0931, "num_tokens": 1412185382.0, "reward": 0.5379464626312256, "reward_std": 0.11569132842123508, "rewards/accuracy_reward/mean": 0.0446428582072258, "rewards/accuracy_reward/std": 0.19926396012306213, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767448961735, "step": 2958 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 846.6138763427734, "completions/mean_terminated_length": 754.7957611083984, "completions/min_length": 301.25, "completions/min_terminated_length": 301.25, "epoch": 0.8838772309760287, "grad_norm": 0.3714568614959717, "kl": 2.4375, "learning_rate": 3.464112755815119e-07, "loss": 0.1233, "num_tokens": 1412641465.0, "reward": 0.6729911118745804, "reward_std": 0.21009713038802147, "rewards/accuracy_reward/mean": 0.18861607182770967, "rewards/accuracy_reward/std": 0.3635432794690132, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.046347017865628004, "step": 2959 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.25, "completions/mean_length": 877.982177734375, "completions/mean_terminated_length": 768.5289459228516, "completions/min_length": 324.25, "completions/min_terminated_length": 324.25, "epoch": 0.8841759390635502, "grad_norm": 0.23938919603824615, "kl": 2.7578125, "learning_rate": 3.435707611964545e-07, "loss": 0.1419, "num_tokens": 1413113201.0, "reward": 0.5948660969734192, "reward_std": 0.13504582457244396, "rewards/accuracy_reward/mean": 0.10714285844005644, "rewards/accuracy_reward/std": 0.27261132560670376, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.053949310444295406, "step": 2960 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 877.5022735595703, "completions/mean_terminated_length": 777.0223541259766, "completions/min_length": 442.25, "completions/min_terminated_length": 442.25, "epoch": 0.8844746471510716, "grad_norm": 0.18190665543079376, "kl": 1.90234375, "learning_rate": 3.4074173710931804e-07, "loss": 0.098, "num_tokens": 1413572258.0, "reward": 0.6328125298023224, "reward_std": 0.10373544413596392, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.28023769706487656, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04326625540852547, "step": 2961 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3526785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 833.2210083007812, "completions/mean_terminated_length": 730.0846557617188, "completions/min_length": 255.75, "completions/min_terminated_length": 255.75, "epoch": 0.8847733552385931, "grad_norm": 0.27922049164772034, "kl": 1.958984375, "learning_rate": 3.379242066863886e-07, "loss": 0.1208, "num_tokens": 1414013461.0, "reward": 0.740513414144516, "reward_std": 0.19762663170695305, "rewards/accuracy_reward/mean": 0.2500000037252903, "rewards/accuracy_reward/std": 0.4273851662874222, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 2962 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.0, "completions/mean_length": 857.1116333007812, "completions/mean_terminated_length": 709.1005706787109, "completions/min_length": 355.75, "completions/min_terminated_length": 355.75, "epoch": 0.8850720633261145, "grad_norm": 0.30980631709098816, "kl": 1.900390625, "learning_rate": 3.351181732802811e-07, "loss": 0.1028, "num_tokens": 1414463287.0, "reward": 0.5809152126312256, "reward_std": 0.15714368969202042, "rewards/accuracy_reward/mean": 0.08928571571595967, "rewards/accuracy_reward/std": 0.2436857596039772, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.044545551761984825, "step": 2963 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 815.1250305175781, "completions/mean_terminated_length": 688.4145660400391, "completions/min_length": 275.25, "completions/min_terminated_length": 275.25, "epoch": 0.8853707714136361, "grad_norm": 0.4003291130065918, "kl": 2.458984375, "learning_rate": 3.323236402299246e-07, "loss": 0.1267, "num_tokens": 1414898511.0, "reward": 0.679129496216774, "reward_std": 0.17353500425815582, "rewards/accuracy_reward/mean": 0.1990327350795269, "rewards/accuracy_reward/std": 0.3910369277000427, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04994932655245066, "step": 2964 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 863.6741333007812, "completions/mean_terminated_length": 746.0457916259766, "completions/min_length": 388.0, "completions/min_terminated_length": 388.0, "epoch": 0.8856694795011575, "grad_norm": 0.35616493225097656, "kl": 2.318359375, "learning_rate": 3.2954061086056924e-07, "loss": 0.1239, "num_tokens": 1415358685.0, "reward": 0.6986607536673546, "reward_std": 0.10152044426649809, "rewards/accuracy_reward/mean": 0.207589291036129, "rewards/accuracy_reward/std": 0.3363794535398483, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 2965 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 862.2187805175781, "completions/mean_terminated_length": 737.0761566162109, "completions/min_length": 371.75, "completions/min_terminated_length": 371.75, "epoch": 0.885968187588679, "grad_norm": 0.45922598242759705, "kl": 2.267578125, "learning_rate": 3.2676908848377263e-07, "loss": 0.1267, "num_tokens": 1415807615.0, "reward": 0.7315848618745804, "reward_std": 0.12770690582692623, "rewards/accuracy_reward/mean": 0.2410714291036129, "rewards/accuracy_reward/std": 0.4185744598507881, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241046637297, "step": 2966 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 891.622802734375, "completions/mean_terminated_length": 768.6091918945312, "completions/min_length": 375.75, "completions/min_terminated_length": 375.75, "epoch": 0.8862668956762004, "grad_norm": 0.4969117343425751, "kl": 2.90234375, "learning_rate": 3.2400907639740243e-07, "loss": 0.1466, "num_tokens": 1416276614.0, "reward": 0.581473246216774, "reward_std": 0.11235336819663644, "rewards/accuracy_reward/mean": 0.09375000116415322, "rewards/accuracy_reward/std": 0.2178180180490017, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05349547974765301, "step": 2967 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 831.3549499511719, "completions/mean_terminated_length": 688.3413848876953, "completions/min_length": 276.75, "completions/min_terminated_length": 276.75, "epoch": 0.886565603763722, "grad_norm": 0.2751419246196747, "kl": 3.5, "learning_rate": 3.2126057788562926e-07, "loss": 0.1972, "num_tokens": 1416719861.0, "reward": 0.6372768208384514, "reward_std": 0.15121035743504763, "rewards/accuracy_reward/mean": 0.15178571082651615, "rewards/accuracy_reward/std": 0.2945944294333458, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.057715265080332756, "step": 2968 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 881.7924499511719, "completions/mean_terminated_length": 770.6110076904297, "completions/min_length": 245.75, "completions/min_terminated_length": 245.75, "epoch": 0.8868643118512434, "grad_norm": 0.2696394920349121, "kl": 1.6982421875, "learning_rate": 3.185235962189237e-07, "loss": 0.072, "num_tokens": 1417186328.0, "reward": 0.5613839626312256, "reward_std": 0.08976731356233358, "rewards/accuracy_reward/mean": 0.06919642677530646, "rewards/accuracy_reward/std": 0.20650966465473175, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.0374377416446805, "step": 2969 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41741071428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 839.4241638183594, "completions/mean_terminated_length": 705.0021514892578, "completions/min_length": 276.75, "completions/min_terminated_length": 276.75, "epoch": 0.8871630199387649, "grad_norm": 0.42040395736694336, "kl": 1.830078125, "learning_rate": 3.1579813465405064e-07, "loss": 0.0913, "num_tokens": 1417627558.0, "reward": 0.6774553954601288, "reward_std": 0.133401830913499, "rewards/accuracy_reward/mean": 0.1852678619325161, "rewards/accuracy_reward/std": 0.35776337236166, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041067422833293676, "step": 2970 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3571428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 852.1451110839844, "completions/mean_terminated_length": 763.1195831298828, "completions/min_length": 337.25, "completions/min_terminated_length": 337.25, "epoch": 0.8874617280262863, "grad_norm": 0.21881665289402008, "kl": 2.255859375, "learning_rate": 3.1308419643406915e-07, "loss": 0.1265, "num_tokens": 1418085527.0, "reward": 0.6785714626312256, "reward_std": 0.15277119912207127, "rewards/accuracy_reward/mean": 0.1897321417927742, "rewards/accuracy_reward/std": 0.3899621367454529, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.051574116572737694, "step": 2971 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5133928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 900.3013916015625, "completions/mean_terminated_length": 766.0774383544922, "completions/min_length": 406.25, "completions/min_terminated_length": 406.25, "epoch": 0.8877604361138077, "grad_norm": 0.36392077803611755, "kl": 2.236328125, "learning_rate": 3.103817847883273e-07, "loss": 0.1115, "num_tokens": 1418559726.0, "reward": 0.604910746216774, "reward_std": 0.17599990591406822, "rewards/accuracy_reward/mean": 0.11607142724096775, "rewards/accuracy_reward/std": 0.31650567427277565, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982165873051, "step": 2972 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 905.4732666015625, "completions/mean_terminated_length": 785.1727447509766, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.8880591442013293, "grad_norm": 0.3317025303840637, "kl": 2.4921875, "learning_rate": 3.076909029324571e-07, "loss": 0.1213, "num_tokens": 1419037074.0, "reward": 0.6138393133878708, "reward_std": 0.1873246543109417, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.3240918405354023, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.05330301355570555, "step": 2973 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3013392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 814.810302734375, "completions/mean_terminated_length": 729.5452270507812, "completions/min_length": 356.75, "completions/min_terminated_length": 356.75, "epoch": 0.8883578522888507, "grad_norm": 0.29204410314559937, "kl": 2.16796875, "learning_rate": 3.0501155406836623e-07, "loss": 0.1209, "num_tokens": 1419472765.0, "reward": 0.7187500298023224, "reward_std": 0.166574003174901, "rewards/accuracy_reward/mean": 0.24479166278615594, "rewards/accuracy_reward/std": 0.36831431463360786, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04777701571583748, "step": 2974 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3214285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 825.5335083007812, "completions/mean_terminated_length": 738.86181640625, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 0.8886565603763722, "grad_norm": 0.3717530071735382, "kl": 2.412109375, "learning_rate": 3.023437413842478e-07, "loss": 0.1388, "num_tokens": 1419910012.0, "reward": 0.6941964626312256, "reward_std": 0.19878572970628738, "rewards/accuracy_reward/mean": 0.2053571455180645, "rewards/accuracy_reward/std": 0.3911844417452812, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104431241751, "step": 2975 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 857.2277069091797, "completions/mean_terminated_length": 731.2154846191406, "completions/min_length": 357.5, "completions/min_terminated_length": 357.5, "epoch": 0.8889552684638936, "grad_norm": 0.2844732701778412, "kl": 2.111328125, "learning_rate": 2.996874680545603e-07, "loss": 0.1082, "num_tokens": 1420361426.0, "reward": 0.7873884290456772, "reward_std": 0.16663040034472942, "rewards/accuracy_reward/mean": 0.2968750037252903, "rewards/accuracy_reward/std": 0.4201724901795387, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 2976 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3236607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 835.0134429931641, "completions/mean_terminated_length": 748.303955078125, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 0.8892539765514151, "grad_norm": 0.2964181900024414, "kl": 1.865234375, "learning_rate": 2.970427372400353e-07, "loss": 0.1152, "num_tokens": 1420807992.0, "reward": 0.6562500447034836, "reward_std": 0.17859460227191448, "rewards/accuracy_reward/mean": 0.16517857275903225, "rewards/accuracy_reward/std": 0.3671586290001869, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529812000691891, "step": 2977 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 815.9955596923828, "completions/mean_terminated_length": 708.9426879882812, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "epoch": 0.8895526846389366, "grad_norm": 0.27553269267082214, "kl": 2.66796875, "learning_rate": 2.9440955208767e-07, "loss": 0.1522, "num_tokens": 1421238054.0, "reward": 0.7946428805589676, "reward_std": 0.22277633100748062, "rewards/accuracy_reward/mean": 0.30580357275903225, "rewards/accuracy_reward/std": 0.4139651283621788, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.051717888563871384, "step": 2978 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 857.669677734375, "completions/mean_terminated_length": 742.0376586914062, "completions/min_length": 347.5, "completions/min_terminated_length": 347.5, "epoch": 0.8898513927264581, "grad_norm": 0.5992572903633118, "kl": 2.63671875, "learning_rate": 2.9178791573071907e-07, "loss": 0.1644, "num_tokens": 1421689378.0, "reward": 0.6428571790456772, "reward_std": 0.20124400034546852, "rewards/accuracy_reward/mean": 0.15625000232830644, "rewards/accuracy_reward/std": 0.33116183802485466, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071492433548, "rewards/tag_count_reward/std": 0.05632450245320797, "step": 2979 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 883.7120971679688, "completions/mean_terminated_length": 743.0694274902344, "completions/min_length": 274.75, "completions/min_terminated_length": 274.75, "epoch": 0.8901501008139795, "grad_norm": 0.3718980550765991, "kl": 2.9765625, "learning_rate": 2.8917783128870167e-07, "loss": 0.1603, "num_tokens": 1422163761.0, "reward": 0.6127232313156128, "reward_std": 0.118944869376719, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.3315972164273262, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058050588704645634, "step": 2980 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 855.2924499511719, "completions/mean_terminated_length": 758.1931457519531, "completions/min_length": 244.0, "completions/min_terminated_length": 244.0, "epoch": 0.890448808901501, "grad_norm": 0.2468739002943039, "kl": 2.375, "learning_rate": 2.865793018673857e-07, "loss": 0.1172, "num_tokens": 1422612612.0, "reward": 0.6099330633878708, "reward_std": 0.15069221518933773, "rewards/accuracy_reward/mean": 0.12053571548312902, "rewards/accuracy_reward/std": 0.31048179790377617, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04940675385296345, "step": 2981 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 868.325927734375, "completions/mean_terminated_length": 763.971435546875, "completions/min_length": 359.75, "completions/min_terminated_length": 359.75, "epoch": 0.8907475169890224, "grad_norm": 0.4191649854183197, "kl": 2.37109375, "learning_rate": 2.8399233055879327e-07, "loss": 0.1436, "num_tokens": 1423074038.0, "reward": 0.7639509290456772, "reward_std": 0.20152486115694046, "rewards/accuracy_reward/mean": 0.2745535746216774, "rewards/accuracy_reward/std": 0.44209569692611694, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.05005982704460621, "step": 2982 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 911.5692291259766, "completions/mean_terminated_length": 801.0285339355469, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.891046225076544, "grad_norm": 0.43442997336387634, "kl": 2.4140625, "learning_rate": 2.8141692044118874e-07, "loss": 0.1152, "num_tokens": 1423553221.0, "reward": 0.6746652126312256, "reward_std": 0.23312178254127502, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.3456191346049309, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.052855560556054115, "step": 2983 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45758928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 895.2254791259766, "completions/mean_terminated_length": 786.147705078125, "completions/min_length": 401.25, "completions/min_terminated_length": 401.25, "epoch": 0.8913449331640654, "grad_norm": 0.3092375695705414, "kl": 2.35546875, "learning_rate": 2.788530745790874e-07, "loss": 0.1193, "num_tokens": 1424030698.0, "reward": 0.6462053805589676, "reward_std": 0.1507047712802887, "rewards/accuracy_reward/mean": 0.15624999720603228, "rewards/accuracy_reward/std": 0.2952676862478256, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.045422971714287996, "step": 2984 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37500000000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.5, "completions/mean_length": 845.3683319091797, "completions/mean_terminated_length": 737.6694946289062, "completions/min_length": 349.25, "completions/min_terminated_length": 349.25, "epoch": 0.8916436412515869, "grad_norm": 0.4076472520828247, "kl": 2.8671875, "learning_rate": 2.7630079602323447e-07, "loss": 0.1684, "num_tokens": 1424476239.0, "reward": 0.615513414144516, "reward_std": 0.15304986014962196, "rewards/accuracy_reward/mean": 0.12946428847499192, "rewards/accuracy_reward/std": 0.3051716797053814, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491156578064, "rewards/tag_count_reward/std": 0.05626536998897791, "step": 2985 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 875.4978179931641, "completions/mean_terminated_length": 749.8066253662109, "completions/min_length": 354.75, "completions/min_terminated_length": 354.75, "epoch": 0.8919423493391083, "grad_norm": 0.24617262184619904, "kl": 1.900390625, "learning_rate": 2.7376008781061835e-07, "loss": 0.0968, "num_tokens": 1424945150.0, "reward": 0.6378348469734192, "reward_std": 0.14017530344426632, "rewards/accuracy_reward/mean": 0.1450892873108387, "rewards/accuracy_reward/std": 0.3535059317946434, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455261349678, "rewards/tag_count_reward/std": 0.04090118408203125, "step": 2986 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 849.8795013427734, "completions/mean_terminated_length": 725.2356109619141, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.8922410574266298, "grad_norm": 0.39695313572883606, "kl": 2.822265625, "learning_rate": 2.7123095296445743e-07, "loss": 0.1517, "num_tokens": 1425401752.0, "reward": 0.6104911118745804, "reward_std": 0.15315939486026764, "rewards/accuracy_reward/mean": 0.12276785750873387, "rewards/accuracy_reward/std": 0.2830950375646353, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.052990143187344074, "step": 2987 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 854.2857513427734, "completions/mean_terminated_length": 739.16357421875, "completions/min_length": 337.25, "completions/min_terminated_length": 337.25, "epoch": 0.8925397655141513, "grad_norm": 0.2721177637577057, "kl": 2.75390625, "learning_rate": 2.6871339449419995e-07, "loss": 0.1386, "num_tokens": 1425854984.0, "reward": 0.5987723469734192, "reward_std": 0.13536526495590806, "rewards/accuracy_reward/mean": 0.1138392835855484, "rewards/accuracy_reward/std": 0.269532173871994, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05874948389828205, "step": 2988 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.75, "completions/mean_length": 861.7455902099609, "completions/mean_terminated_length": 747.4670867919922, "completions/min_length": 212.5, "completions/min_terminated_length": 212.5, "epoch": 0.8928384736016728, "grad_norm": 0.4126955568790436, "kl": 1.880859375, "learning_rate": 2.662074153955152e-07, "loss": 0.1112, "num_tokens": 1426318342.0, "reward": 0.6210937798023224, "reward_std": 0.13062646985054016, "rewards/accuracy_reward/mean": 0.12946428847499192, "rewards/accuracy_reward/std": 0.30394077859818935, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04603219963610172, "step": 2989 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 860.4643249511719, "completions/mean_terminated_length": 736.4018859863281, "completions/min_length": 356.75, "completions/min_terminated_length": 356.75, "epoch": 0.8931371816891942, "grad_norm": 0.22427310049533844, "kl": 1.94140625, "learning_rate": 2.637130186503001e-07, "loss": 0.0895, "num_tokens": 1426781062.0, "reward": 0.6216518133878708, "reward_std": 0.1363210417330265, "rewards/accuracy_reward/mean": 0.13169642724096775, "rewards/accuracy_reward/std": 0.3221092112362385, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.047033360693603754, "step": 2990 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 869.2857513427734, "completions/mean_terminated_length": 766.4889678955078, "completions/min_length": 405.75, "completions/min_terminated_length": 405.75, "epoch": 0.8934358897767157, "grad_norm": 0.25607866048812866, "kl": 1.703125, "learning_rate": 2.612302072266637e-07, "loss": 0.0933, "num_tokens": 1427238246.0, "reward": 0.663504496216774, "reward_std": 0.1677896399050951, "rewards/accuracy_reward/mean": 0.1696428582072258, "rewards/accuracy_reward/std": 0.36192087829113007, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 2991 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 837.2522735595703, "completions/mean_terminated_length": 741.2956695556641, "completions/min_length": 382.0, "completions/min_terminated_length": 382.0, "epoch": 0.8937345978642371, "grad_norm": 0.4979933798313141, "kl": 2.12109375, "learning_rate": 2.587589840789351e-07, "loss": 0.1093, "num_tokens": 1427686343.0, "reward": 0.7516741454601288, "reward_std": 0.19732817634940147, "rewards/accuracy_reward/mean": 0.2611607164144516, "rewards/accuracy_reward/std": 0.4330882132053375, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04791746288537979, "step": 2992 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 873.2388763427734, "completions/mean_terminated_length": 758.2144317626953, "completions/min_length": 366.0, "completions/min_terminated_length": 366.0, "epoch": 0.8940333059517587, "grad_norm": 0.2955649495124817, "kl": 2.89306640625, "learning_rate": 2.5629935214764866e-07, "loss": 0.1412, "num_tokens": 1428153778.0, "reward": 0.585937537252903, "reward_std": 0.13536114059388638, "rewards/accuracy_reward/mean": 0.0982142835855484, "rewards/accuracy_reward/std": 0.24608440324664116, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.04386463761329651, "step": 2993 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49553571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 870.2165374755859, "completions/mean_terminated_length": 715.9932861328125, "completions/min_length": 319.0, "completions/min_terminated_length": 319.0, "epoch": 0.8943320140392801, "grad_norm": 0.3716602921485901, "kl": 2.205078125, "learning_rate": 2.5385131435955e-07, "loss": 0.1126, "num_tokens": 1428623523.0, "reward": 0.6495536118745804, "reward_std": 0.1423679105937481, "rewards/accuracy_reward/mean": 0.1584821417927742, "rewards/accuracy_reward/std": 0.36542578786611557, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04560601245611906, "step": 2994 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3348214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 825.9018096923828, "completions/mean_terminated_length": 733.0834808349609, "completions/min_length": 376.25, "completions/min_terminated_length": 376.25, "epoch": 0.8946307221268016, "grad_norm": 0.2393019050359726, "kl": 2.32421875, "learning_rate": 2.51414873627589e-07, "loss": 0.1243, "num_tokens": 1429064551.0, "reward": 0.6015625447034836, "reward_std": 0.19066563807427883, "rewards/accuracy_reward/mean": 0.1116071417927742, "rewards/accuracy_reward/std": 0.3068355917930603, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.049088423140347004, "step": 2995 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 868.9866485595703, "completions/mean_terminated_length": 747.3272857666016, "completions/min_length": 309.0, "completions/min_terminated_length": 309.0, "epoch": 0.894929430214323, "grad_norm": 0.336417019367218, "kl": 1.994140625, "learning_rate": 2.489900328509154e-07, "loss": 0.1047, "num_tokens": 1429534993.0, "reward": 0.6629464775323868, "reward_std": 0.173227671533823, "rewards/accuracy_reward/mean": 0.17187500186264515, "rewards/accuracy_reward/std": 0.368463397026062, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04620361328125, "step": 2996 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 876.9687957763672, "completions/mean_terminated_length": 766.7934875488281, "completions/min_length": 310.25, "completions/min_terminated_length": 310.25, "epoch": 0.8952281383018446, "grad_norm": 0.26720568537712097, "kl": 2.974609375, "learning_rate": 2.465767949148734e-07, "loss": 0.1465, "num_tokens": 1430000787.0, "reward": 0.544084832072258, "reward_std": 0.1297737993299961, "rewards/accuracy_reward/mean": 0.05803571455180645, "rewards/accuracy_reward/std": 0.2197403945028782, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05411295732483268, "step": 2997 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5580357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 926.1518096923828, "completions/mean_terminated_length": 809.3955078125, "completions/min_length": 534.75, "completions/min_terminated_length": 534.75, "epoch": 0.895526846389366, "grad_norm": 0.23336978256702423, "kl": 2.7421875, "learning_rate": 2.4417516269100496e-07, "loss": 0.1311, "num_tokens": 1430487127.0, "reward": 0.6350446864962578, "reward_std": 0.13329939357936382, "rewards/accuracy_reward/mean": 0.1473214291036129, "rewards/accuracy_reward/std": 0.2992316260933876, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 2998 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 820.8594055175781, "completions/mean_terminated_length": 691.5464782714844, "completions/min_length": 278.25, "completions/min_terminated_length": 278.25, "epoch": 0.8958255544768875, "grad_norm": 0.3711952865123749, "kl": 3.55859375, "learning_rate": 2.4178513903703847e-07, "loss": 0.1975, "num_tokens": 1430927144.0, "reward": 0.6194196790456772, "reward_std": 0.13912342116236687, "rewards/accuracy_reward/mean": 0.13392856903374195, "rewards/accuracy_reward/std": 0.3412448838353157, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.058666424825787544, "step": 2999 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 841.1272583007812, "completions/mean_terminated_length": 732.250732421875, "completions/min_length": 330.75, "completions/min_terminated_length": 330.75, "epoch": 0.8961242625644089, "grad_norm": 0.24868997931480408, "kl": 2.42578125, "learning_rate": 2.394067267968925e-07, "loss": 0.13, "num_tokens": 1431374305.0, "reward": 0.7131696790456772, "reward_std": 0.20462295785546303, "rewards/accuracy_reward/mean": 0.232886902987957, "rewards/accuracy_reward/std": 0.4149372726678848, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04903263598680496, "step": 3000 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 874.6540679931641, "completions/mean_terminated_length": 769.9593658447266, "completions/min_length": 382.5, "completions/min_terminated_length": 382.5, "epoch": 0.8964229706519304, "grad_norm": 0.3047565221786499, "kl": 2.515625, "learning_rate": 2.370399288006664e-07, "loss": 0.1364, "num_tokens": 1431842566.0, "reward": 0.706473246216774, "reward_std": 0.19698704220354557, "rewards/accuracy_reward/mean": 0.21874999813735485, "rewards/accuracy_reward/std": 0.38402874022722244, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 3001 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 860.6897735595703, "completions/mean_terminated_length": 744.0604248046875, "completions/min_length": 394.75, "completions/min_terminated_length": 394.75, "epoch": 0.8967216787394519, "grad_norm": 0.3584211766719818, "kl": 2.8125, "learning_rate": 2.346847478646419e-07, "loss": 0.1633, "num_tokens": 1432300699.0, "reward": 0.698660746216774, "reward_std": 0.20417429879307747, "rewards/accuracy_reward/mean": 0.2098214291036129, "rewards/accuracy_reward/std": 0.38621851429343224, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.048379197251051664, "step": 3002 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 859.6272735595703, "completions/mean_terminated_length": 751.2320556640625, "completions/min_length": 402.75, "completions/min_terminated_length": 402.75, "epoch": 0.8970203868269734, "grad_norm": 0.24558211863040924, "kl": 1.919921875, "learning_rate": 2.3234118679127615e-07, "loss": 0.1144, "num_tokens": 1432758628.0, "reward": 0.6646205484867096, "reward_std": 0.16294797137379646, "rewards/accuracy_reward/mean": 0.1718749962747097, "rewards/accuracy_reward/std": 0.36107124015688896, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03955313144251704, "step": 3003 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 872.8259429931641, "completions/mean_terminated_length": 769.314208984375, "completions/min_length": 312.25, "completions/min_terminated_length": 312.25, "epoch": 0.8973190949144948, "grad_norm": 0.35378435254096985, "kl": 2.111328125, "learning_rate": 2.300092483691996e-07, "loss": 0.1045, "num_tokens": 1433216758.0, "reward": 0.601004496216774, "reward_std": 0.17841445095837116, "rewards/accuracy_reward/mean": 0.109375, "rewards/accuracy_reward/std": 0.30264266580343246, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 3004 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3482142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.0, "completions/mean_length": 829.154052734375, "completions/mean_terminated_length": 726.4923706054688, "completions/min_length": 300.5, "completions/min_terminated_length": 300.5, "epoch": 0.8976178030020163, "grad_norm": 0.27108699083328247, "kl": 1.4462890625, "learning_rate": 2.2768893537321145e-07, "loss": 0.0724, "num_tokens": 1433657963.0, "reward": 0.611607164144516, "reward_std": 0.1119515192694962, "rewards/accuracy_reward/mean": 0.1183035671710968, "rewards/accuracy_reward/std": 0.2618259862065315, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 3005 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1001.0, "completions/mean_length": 807.5513763427734, "completions/mean_terminated_length": 671.1406402587891, "completions/min_length": 317.75, "completions/min_terminated_length": 317.75, "epoch": 0.8979165110895377, "grad_norm": 0.3058457374572754, "kl": 3.87890625, "learning_rate": 2.2538025056428216e-07, "loss": 0.2224, "num_tokens": 1434096418.0, "reward": 0.7148437649011612, "reward_std": 0.17333455570042133, "rewards/accuracy_reward/mean": 0.2299107126891613, "rewards/accuracy_reward/std": 0.4095476120710373, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.057826947420835495, "step": 3006 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 871.1763763427734, "completions/mean_terminated_length": 746.8257446289062, "completions/min_length": 392.0, "completions/min_terminated_length": 392.0, "epoch": 0.8982152191770593, "grad_norm": 0.4202619194984436, "kl": 1.546875, "learning_rate": 2.230831966895408e-07, "loss": 0.0906, "num_tokens": 1434559905.0, "reward": 0.6216518133878708, "reward_std": 0.12777302600443363, "rewards/accuracy_reward/mean": 0.12723214272409678, "rewards/accuracy_reward/std": 0.3246284946799278, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 3007 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 852.529052734375, "completions/mean_terminated_length": 741.9584503173828, "completions/min_length": 285.75, "completions/min_terminated_length": 285.75, "epoch": 0.8985139272645807, "grad_norm": 0.3098394274711609, "kl": 2.4501953125, "learning_rate": 2.2079777648227774e-07, "loss": 0.1285, "num_tokens": 1435014478.0, "reward": 0.6183035895228386, "reward_std": 0.10601280350238085, "rewards/accuracy_reward/mean": 0.12946428847499192, "rewards/accuracy_reward/std": 0.24114802293479443, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.04937528306618333, "step": 3008 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3147321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 843.3080902099609, "completions/mean_terminated_length": 761.5395812988281, "completions/min_length": 384.5, "completions/min_terminated_length": 384.5, "epoch": 0.8988126353521022, "grad_norm": 0.28988945484161377, "kl": 2.470703125, "learning_rate": 2.1852399266194312e-07, "loss": 0.1306, "num_tokens": 1435466360.0, "reward": 0.6077009290456772, "reward_std": 0.1501610055565834, "rewards/accuracy_reward/mean": 0.11830357043072581, "rewards/accuracy_reward/std": 0.29912005737423897, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04960599634796381, "step": 3009 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 879.7210235595703, "completions/mean_terminated_length": 736.7809753417969, "completions/min_length": 269.0, "completions/min_terminated_length": 269.0, "epoch": 0.8991113434396236, "grad_norm": 0.25066670775413513, "kl": 2.837890625, "learning_rate": 2.162618479341394e-07, "loss": 0.1585, "num_tokens": 1435929131.0, "reward": 0.706473246216774, "reward_std": 0.2650722414255142, "rewards/accuracy_reward/mean": 0.2187499962747097, "rewards/accuracy_reward/std": 0.4081305041909218, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05313391424715519, "step": 3010 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 808.7031707763672, "completions/mean_terminated_length": 689.0978240966797, "completions/min_length": 276.75, "completions/min_terminated_length": 276.75, "epoch": 0.8994100515271451, "grad_norm": 0.36796656250953674, "kl": 2.427734375, "learning_rate": 2.140113449906167e-07, "loss": 0.1491, "num_tokens": 1436364790.0, "reward": 0.6378348469734192, "reward_std": 0.11334382928907871, "rewards/accuracy_reward/mean": 0.14732143003493547, "rewards/accuracy_reward/std": 0.323543768376112, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047210452146828175, "step": 3011 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 860.2723693847656, "completions/mean_terminated_length": 750.5614318847656, "completions/min_length": 295.25, "completions/min_terminated_length": 295.25, "epoch": 0.8997087596146666, "grad_norm": 0.19897295534610748, "kl": 2.31640625, "learning_rate": 2.117724865092774e-07, "loss": 0.1164, "num_tokens": 1436826880.0, "reward": 0.7315848469734192, "reward_std": 0.15708686225116253, "rewards/accuracy_reward/mean": 0.24107142724096775, "rewards/accuracy_reward/std": 0.4109710305929184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 3012 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3392857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 838.5379791259766, "completions/mean_terminated_length": 741.5740509033203, "completions/min_length": 392.75, "completions/min_terminated_length": 392.75, "epoch": 0.9000074677021881, "grad_norm": 0.304704487323761, "kl": 2.76171875, "learning_rate": 2.0954527515416156e-07, "loss": 0.1548, "num_tokens": 1437275041.0, "reward": 0.6445312798023224, "reward_std": 0.17404834926128387, "rewards/accuracy_reward/mean": 0.16257440485060215, "rewards/accuracy_reward/std": 0.3640722930431366, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05223577655851841, "step": 3013 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 864.9286193847656, "completions/mean_terminated_length": 764.1628875732422, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "epoch": 0.9003061757897095, "grad_norm": 0.31033769249916077, "kl": 2.18359375, "learning_rate": 2.0732971357545707e-07, "loss": 0.1256, "num_tokens": 1437739937.0, "reward": 0.631138414144516, "reward_std": 0.1231840830296278, "rewards/accuracy_reward/mean": 0.140625, "rewards/accuracy_reward/std": 0.292213536798954, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04737457446753979, "step": 3014 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 882.4531555175781, "completions/mean_terminated_length": 790.7837829589844, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "epoch": 0.9006048838772309, "grad_norm": 0.25157108902931213, "kl": 1.427734375, "learning_rate": 2.0512580440948615e-07, "loss": 0.076, "num_tokens": 1438206508.0, "reward": 0.6696428954601288, "reward_std": 0.19034656323492527, "rewards/accuracy_reward/mean": 0.17633928544819355, "rewards/accuracy_reward/std": 0.34899885952472687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 3015 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 869.716552734375, "completions/mean_terminated_length": 735.1408233642578, "completions/min_length": 325.75, "completions/min_terminated_length": 325.75, "epoch": 0.9009035919647524, "grad_norm": 0.25942376255989075, "kl": 3.26953125, "learning_rate": 2.0293355027870554e-07, "loss": 0.1704, "num_tokens": 1438673821.0, "reward": 0.678013414144516, "reward_std": 0.21747731789946556, "rewards/accuracy_reward/mean": 0.2023809514939785, "rewards/accuracy_reward/std": 0.4002191200852394, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4838169664144516, "rewards/tag_count_reward/std": 0.06126142106950283, "step": 3016 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 841.4085235595703, "completions/mean_terminated_length": 718.2412109375, "completions/min_length": 262.75, "completions/min_terminated_length": 262.75, "epoch": 0.9012023000522739, "grad_norm": 0.37061676383018494, "kl": 2.611328125, "learning_rate": 2.0075295379170413e-07, "loss": 0.1423, "num_tokens": 1439130548.0, "reward": 0.672433078289032, "reward_std": 0.18901422806084156, "rewards/accuracy_reward/mean": 0.18340773764066398, "rewards/accuracy_reward/std": 0.325373288244009, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.047950050327926874, "step": 3017 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35937500000000006, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 823.3482513427734, "completions/mean_terminated_length": 712.7784729003906, "completions/min_length": 360.0, "completions/min_terminated_length": 360.0, "epoch": 0.9015010081397954, "grad_norm": 0.36586275696754456, "kl": 2.20703125, "learning_rate": 1.9858401754319967e-07, "loss": 0.1306, "num_tokens": 1439571520.0, "reward": 0.6941964626312256, "reward_std": 0.17692843452095985, "rewards/accuracy_reward/mean": 0.2031250037252903, "rewards/accuracy_reward/std": 0.4002072438597679, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 3018 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 872.3549499511719, "completions/mean_terminated_length": 757.4353637695312, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.9017997162273168, "grad_norm": 0.2734953761100769, "kl": 2.326171875, "learning_rate": 1.9642674411403328e-07, "loss": 0.1226, "num_tokens": 1440034031.0, "reward": 0.6194196790456772, "reward_std": 0.14809714630246162, "rewards/accuracy_reward/mean": 0.12946428544819355, "rewards/accuracy_reward/std": 0.32534296065568924, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04758457001298666, "step": 3019 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37276785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 865.4598693847656, "completions/mean_terminated_length": 774.3565216064453, "completions/min_length": 372.75, "completions/min_terminated_length": 372.75, "epoch": 0.9020984243148383, "grad_norm": 0.4631364941596985, "kl": 1.9462890625, "learning_rate": 1.942811360711705e-07, "loss": 0.1257, "num_tokens": 1440499133.0, "reward": 0.8046875447034836, "reward_std": 0.23552310094237328, "rewards/accuracy_reward/mean": 0.3210565485060215, "rewards/accuracy_reward/std": 0.44434382021427155, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.0421724752523005, "step": 3020 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 845.435302734375, "completions/mean_terminated_length": 716.9391021728516, "completions/min_length": 231.5, "completions/min_terminated_length": 231.5, "epoch": 0.9023971324023597, "grad_norm": 0.4491358697414398, "kl": 2.33984375, "learning_rate": 1.921471959676957e-07, "loss": 0.1257, "num_tokens": 1440952112.0, "reward": 0.6216518133878708, "reward_std": 0.16349021345376968, "rewards/accuracy_reward/mean": 0.12946428451687098, "rewards/accuracy_reward/std": 0.3246024549007416, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04182914597913623, "step": 3021 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 830.2768249511719, "completions/mean_terminated_length": 694.1176300048828, "completions/min_length": 250.25, "completions/min_terminated_length": 250.25, "epoch": 0.9026958404898813, "grad_norm": 0.29377371072769165, "kl": 2.5615234375, "learning_rate": 1.900249263428089e-07, "loss": 0.1534, "num_tokens": 1441396172.0, "reward": 0.679129496216774, "reward_std": 0.15697372145950794, "rewards/accuracy_reward/mean": 0.18749999813735485, "rewards/accuracy_reward/std": 0.38164056092500687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04288960574194789, "step": 3022 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 846.888427734375, "completions/mean_terminated_length": 743.5496063232422, "completions/min_length": 404.5, "completions/min_terminated_length": 404.5, "epoch": 0.9029945485774027, "grad_norm": 0.49640312790870667, "kl": 2.734375, "learning_rate": 1.8791432972182443e-07, "loss": 0.1587, "num_tokens": 1441848714.0, "reward": 0.6021205633878708, "reward_std": 0.146652577444911, "rewards/accuracy_reward/mean": 0.1138392873108387, "rewards/accuracy_reward/std": 0.3087628036737442, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812425494194, "rewards/tag_count_reward/std": 0.051475852727890015, "step": 3023 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 845.232177734375, "completions/mean_terminated_length": 721.7666778564453, "completions/min_length": 336.25, "completions/min_terminated_length": 336.25, "epoch": 0.9032932566649242, "grad_norm": 0.3415190875530243, "kl": 3.24609375, "learning_rate": 1.8581540861616453e-07, "loss": 0.1719, "num_tokens": 1442304674.0, "reward": 0.6021205484867096, "reward_std": 0.11662994418293238, "rewards/accuracy_reward/mean": 0.11607143003493547, "rewards/accuracy_reward/std": 0.24661202728748322, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05585796106606722, "step": 3024 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 834.810302734375, "completions/mean_terminated_length": 723.7692108154297, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "epoch": 0.9035919647524456, "grad_norm": 0.22714821994304657, "kl": 1.90283203125, "learning_rate": 1.8372816552336025e-07, "loss": 0.1099, "num_tokens": 1442747181.0, "reward": 0.765066996216774, "reward_std": 0.19257069751620293, "rewards/accuracy_reward/mean": 0.2782738134264946, "rewards/accuracy_reward/std": 0.4364416003227234, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.035161727108061314, "step": 3025 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49107142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 895.5357513427734, "completions/mean_terminated_length": 773.8775177001953, "completions/min_length": 374.25, "completions/min_terminated_length": 374.25, "epoch": 0.9038906728399672, "grad_norm": 0.4702445864677429, "kl": 2.029296875, "learning_rate": 1.8165260292704712e-07, "loss": 0.1048, "num_tokens": 1443230093.0, "reward": 0.640066996216774, "reward_std": 0.1678337510675192, "rewards/accuracy_reward/mean": 0.14955357369035482, "rewards/accuracy_reward/std": 0.33833901584148407, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.046577731147408485, "step": 3026 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 853.7924499511719, "completions/mean_terminated_length": 727.3348541259766, "completions/min_length": 283.5, "completions/min_terminated_length": 283.5, "epoch": 0.9041893809274886, "grad_norm": 0.2873724699020386, "kl": 1.939453125, "learning_rate": 1.7958872329696177e-07, "loss": 0.1107, "num_tokens": 1443687824.0, "reward": 0.6796875298023224, "reward_std": 0.1640671957284212, "rewards/accuracy_reward/mean": 0.18973214272409678, "rewards/accuracy_reward/std": 0.35767097026109695, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.047033360693603754, "step": 3027 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 847.2455902099609, "completions/mean_terminated_length": 743.8323822021484, "completions/min_length": 470.5, "completions/min_terminated_length": 470.5, "epoch": 0.9044880890150101, "grad_norm": 0.4246242940425873, "kl": 2.6455078125, "learning_rate": 1.7753652908893636e-07, "loss": 0.1354, "num_tokens": 1444138446.0, "reward": 0.6808035969734192, "reward_std": 0.1303102783858776, "rewards/accuracy_reward/mean": 0.19196428963914514, "rewards/accuracy_reward/std": 0.32966480776667595, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.049176040571182966, "step": 3028 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 846.4129943847656, "completions/mean_terminated_length": 759.5655059814453, "completions/min_length": 414.25, "completions/min_terminated_length": 414.25, "epoch": 0.9047867971025315, "grad_norm": 0.7120989561080933, "kl": 2.115234375, "learning_rate": 1.754960227449032e-07, "loss": 0.103, "num_tokens": 1444583463.0, "reward": 0.6886160969734192, "reward_std": 0.20198892429471016, "rewards/accuracy_reward/mean": 0.19642857648432255, "rewards/accuracy_reward/std": 0.3835926279425621, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.04241547454148531, "step": 3029 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 817.1986846923828, "completions/mean_terminated_length": 696.2283630371094, "completions/min_length": 301.75, "completions/min_terminated_length": 301.75, "epoch": 0.905085505190053, "grad_norm": 0.39668765664100647, "kl": 2.33984375, "learning_rate": 1.734672066928822e-07, "loss": 0.1394, "num_tokens": 1445021808.0, "reward": 0.7271205633878708, "reward_std": 0.18983281590044498, "rewards/accuracy_reward/mean": 0.2366071417927742, "rewards/accuracy_reward/std": 0.423353873193264, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04712030291557312, "step": 3030 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 862.3281707763672, "completions/mean_terminated_length": 733.8901519775391, "completions/min_length": 335.75, "completions/min_terminated_length": 335.75, "epoch": 0.9053842132775745, "grad_norm": 0.29138246178627014, "kl": 2.2587890625, "learning_rate": 1.7145008334698898e-07, "loss": 0.128, "num_tokens": 1445480531.0, "reward": 0.6858258992433548, "reward_std": 0.13904364220798016, "rewards/accuracy_reward/mean": 0.1964285778813064, "rewards/accuracy_reward/std": 0.2912781648337841, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04909886047244072, "step": 3031 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 870.9107513427734, "completions/mean_terminated_length": 773.0597229003906, "completions/min_length": 405.5, "completions/min_terminated_length": 405.5, "epoch": 0.905682921365096, "grad_norm": 0.24759086966514587, "kl": 2.359375, "learning_rate": 1.6944465510741803e-07, "loss": 0.132, "num_tokens": 1445942475.0, "reward": 0.5987723469734192, "reward_std": 0.13921080715954304, "rewards/accuracy_reward/mean": 0.10937499906867743, "rewards/accuracy_reward/std": 0.30752700567245483, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04884427320212126, "step": 3032 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42187499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 845.7388916015625, "completions/mean_terminated_length": 718.0746765136719, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 0.9059816294526174, "grad_norm": 0.3171882927417755, "kl": 2.60546875, "learning_rate": 1.6745092436045495e-07, "loss": 0.1549, "num_tokens": 1446385622.0, "reward": 0.619419664144516, "reward_std": 0.17797316424548626, "rewards/accuracy_reward/mean": 0.12946428824216127, "rewards/accuracy_reward/std": 0.32212312519550323, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04863459337502718, "step": 3033 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3861607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 859.4486999511719, "completions/mean_terminated_length": 761.0844879150391, "completions/min_length": 335.5, "completions/min_terminated_length": 335.5, "epoch": 0.9062803375401389, "grad_norm": 0.28333523869514465, "kl": 2.525390625, "learning_rate": 1.6546889347846095e-07, "loss": 0.131, "num_tokens": 1446845439.0, "reward": 0.7070312798023224, "reward_std": 0.19871891289949417, "rewards/accuracy_reward/mean": 0.21875000558793545, "rewards/accuracy_reward/std": 0.3729393631219864, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918200105429, "step": 3034 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39955357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 831.8661041259766, "completions/mean_terminated_length": 704.0993957519531, "completions/min_length": 208.75, "completions/min_terminated_length": 208.75, "epoch": 0.9065790456276603, "grad_norm": 0.33318087458610535, "kl": 3.8984375, "learning_rate": 1.6349856481987835e-07, "loss": 0.2476, "num_tokens": 1447285875.0, "reward": 0.6880580633878708, "reward_std": 0.21356156468391418, "rewards/accuracy_reward/mean": 0.2053571417927742, "rewards/accuracy_reward/std": 0.36945922672748566, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06215373892337084, "step": 3035 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.25, "completions/mean_length": 887.0848541259766, "completions/mean_terminated_length": 788.7475738525391, "completions/min_length": 448.25, "completions/min_terminated_length": 448.25, "epoch": 0.9068777537151819, "grad_norm": 0.2190503478050232, "kl": 2.8125, "learning_rate": 1.615399407292251e-07, "loss": 0.1447, "num_tokens": 1447745657.0, "reward": 0.5770089402794838, "reward_std": 0.1110528139397502, "rewards/accuracy_reward/mean": 0.09151785401627421, "rewards/accuracy_reward/std": 0.213733721524477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.487723208963871, "rewards/tag_count_reward/std": 0.05360598023980856, "step": 3036 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 856.9442291259766, "completions/mean_terminated_length": 724.2098693847656, "completions/min_length": 386.5, "completions/min_terminated_length": 386.5, "epoch": 0.9071764618027033, "grad_norm": 0.5195897817611694, "kl": 2.310546875, "learning_rate": 1.5959302353709128e-07, "loss": 0.1229, "num_tokens": 1448208832.0, "reward": 0.7890625298023224, "reward_std": 0.19376271218061447, "rewards/accuracy_reward/mean": 0.2991071455180645, "rewards/accuracy_reward/std": 0.4400997832417488, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.049232195131480694, "step": 3037 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38169642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 856.325927734375, "completions/mean_terminated_length": 754.0377960205078, "completions/min_length": 298.25, "completions/min_terminated_length": 298.25, "epoch": 0.9074751698902248, "grad_norm": 0.36025628447532654, "kl": 1.6494140625, "learning_rate": 1.5765781556013493e-07, "loss": 0.0897, "num_tokens": 1448663618.0, "reward": 0.6841518133878708, "reward_std": 0.186025558039546, "rewards/accuracy_reward/mean": 0.18973213993012905, "rewards/accuracy_reward/std": 0.36918532103300095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.034261973574757576, "step": 3038 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 838.0893096923828, "completions/mean_terminated_length": 732.3030548095703, "completions/min_length": 377.75, "completions/min_terminated_length": 377.75, "epoch": 0.9077738779777462, "grad_norm": 0.31814706325531006, "kl": 2.3671875, "learning_rate": 1.5573431910108404e-07, "loss": 0.135, "num_tokens": 1449111050.0, "reward": 0.6356027126312256, "reward_std": 0.169235248118639, "rewards/accuracy_reward/mean": 0.14508928917348385, "rewards/accuracy_reward/std": 0.3476926386356354, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04681241046637297, "step": 3039 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 825.4509429931641, "completions/mean_terminated_length": 707.9283599853516, "completions/min_length": 294.0, "completions/min_terminated_length": 294.0, "epoch": 0.9080725860652678, "grad_norm": 0.32877054810523987, "kl": 2.099609375, "learning_rate": 1.538225364487278e-07, "loss": 0.1072, "num_tokens": 1449547732.0, "reward": 0.6333705484867096, "reward_std": 0.10982183367013931, "rewards/accuracy_reward/mean": 0.14285714458674192, "rewards/accuracy_reward/std": 0.25096989423036575, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.03968940582126379, "step": 3040 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4151785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 858.0044860839844, "completions/mean_terminated_length": 744.9802551269531, "completions/min_length": 204.5, "completions/min_terminated_length": 204.5, "epoch": 0.9083712941527892, "grad_norm": 0.257738322019577, "kl": 1.931640625, "learning_rate": 1.519224698779198e-07, "loss": 0.1018, "num_tokens": 1450000934.0, "reward": 0.5758928805589676, "reward_std": 0.09020718047395349, "rewards/accuracy_reward/mean": 0.08482143119908869, "rewards/accuracy_reward/std": 0.24820689670741558, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 3041 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 852.6004943847656, "completions/mean_terminated_length": 722.341796875, "completions/min_length": 350.25, "completions/min_terminated_length": 350.25, "epoch": 0.9086700022403107, "grad_norm": 0.4144856035709381, "kl": 2.396484375, "learning_rate": 1.5003412164957154e-07, "loss": 0.1406, "num_tokens": 1450455347.0, "reward": 0.7170759290456772, "reward_std": 0.19694490358233452, "rewards/accuracy_reward/mean": 0.22767857275903225, "rewards/accuracy_reward/std": 0.3847702518105507, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810389876366, "step": 3042 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 851.091552734375, "completions/mean_terminated_length": 717.7355804443359, "completions/min_length": 329.0, "completions/min_terminated_length": 329.0, "epoch": 0.9089687103278321, "grad_norm": 0.28794988989830017, "kl": 2.349609375, "learning_rate": 1.4815749401064894e-07, "loss": 0.118, "num_tokens": 1450911676.0, "reward": 0.6819196790456772, "reward_std": 0.19557619839906693, "rewards/accuracy_reward/mean": 0.19196429196745157, "rewards/accuracy_reward/std": 0.36386220157146454, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 3043 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3950892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 848.8549499511719, "completions/mean_terminated_length": 742.1190948486328, "completions/min_length": 265.0, "completions/min_terminated_length": 265.0, "epoch": 0.9092674184153536, "grad_norm": 0.3120889365673065, "kl": 1.5751953125, "learning_rate": 1.4629258919417578e-07, "loss": 0.1065, "num_tokens": 1451362507.0, "reward": 0.7036830633878708, "reward_std": 0.20459149032831192, "rewards/accuracy_reward/mean": 0.20982142398133874, "rewards/accuracy_reward/std": 0.3566025160253048, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 3044 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.32589285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.75, "completions/mean_length": 818.3772583007812, "completions/mean_terminated_length": 721.1335144042969, "completions/min_length": 275.0, "completions/min_terminated_length": 275.0, "epoch": 0.909566126502875, "grad_norm": 0.3677496016025543, "kl": 2.71875, "learning_rate": 1.444394094192225e-07, "loss": 0.1349, "num_tokens": 1451798692.0, "reward": 0.741629496216774, "reward_std": 0.2076556421816349, "rewards/accuracy_reward/mean": 0.2544642798602581, "rewards/accuracy_reward/std": 0.43305375427007675, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05343634728342295, "step": 3045 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3415178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 843.8683471679688, "completions/mean_terminated_length": 747.8360290527344, "completions/min_length": 246.0, "completions/min_terminated_length": 246.0, "epoch": 0.9098648345903966, "grad_norm": 0.22880251705646515, "kl": 1.96875, "learning_rate": 1.4259795689090972e-07, "loss": 0.1017, "num_tokens": 1452255273.0, "reward": 0.5530134215950966, "reward_std": 0.11755036655813456, "rewards/accuracy_reward/mean": 0.06026785564608872, "rewards/accuracy_reward/std": 0.17866151221096516, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 3046 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44642857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 873.3371124267578, "completions/mean_terminated_length": 757.6582336425781, "completions/min_length": 301.25, "completions/min_terminated_length": 301.25, "epoch": 0.910163542677918, "grad_norm": 0.175869420170784, "kl": 1.955078125, "learning_rate": 1.407682338004046e-07, "loss": 0.1042, "num_tokens": 1452716992.0, "reward": 0.6037946790456772, "reward_std": 0.14118049293756485, "rewards/accuracy_reward/mean": 0.11160714738070965, "rewards/accuracy_reward/std": 0.3120334595441818, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04357414972037077, "step": 3047 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.390625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 827.9286193847656, "completions/mean_terminated_length": 708.7471466064453, "completions/min_length": 360.25, "completions/min_terminated_length": 360.25, "epoch": 0.9104622507654395, "grad_norm": 0.32314544916152954, "kl": 1.6171875, "learning_rate": 1.3895024232491338e-07, "loss": 0.1029, "num_tokens": 1453155200.0, "reward": 0.6919643133878708, "reward_std": 0.15599718550220132, "rewards/accuracy_reward/mean": 0.1986607164144516, "rewards/accuracy_reward/std": 0.39838625490665436, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.037730947602540255, "step": 3048 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 877.0335235595703, "completions/mean_terminated_length": 742.5592193603516, "completions/min_length": 287.75, "completions/min_terminated_length": 287.75, "epoch": 0.9107609588529609, "grad_norm": 0.8257846832275391, "kl": 2.484375, "learning_rate": 1.3714398462768563e-07, "loss": 0.1595, "num_tokens": 1453626079.0, "reward": 0.5630580708384514, "reward_std": 0.12076781317591667, "rewards/accuracy_reward/mean": 0.07477678544819355, "rewards/accuracy_reward/std": 0.21000991761684418, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.05020359717309475, "step": 3049 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4933035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 875.0513916015625, "completions/mean_terminated_length": 731.1313171386719, "completions/min_length": 337.0, "completions/min_terminated_length": 337.0, "epoch": 0.9110596669404825, "grad_norm": 0.3741486072540283, "kl": 2.78515625, "learning_rate": 1.3534946285801098e-07, "loss": 0.1505, "num_tokens": 1454098726.0, "reward": 0.6484375298023224, "reward_std": 0.20144779235124588, "rewards/accuracy_reward/mean": 0.16071428824216127, "rewards/accuracy_reward/std": 0.3423375189304352, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05333347246050835, "step": 3050 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 854.5357666015625, "completions/mean_terminated_length": 730.6059112548828, "completions/min_length": 386.75, "completions/min_terminated_length": 386.75, "epoch": 0.9113583750280039, "grad_norm": 0.26741909980773926, "kl": 1.650390625, "learning_rate": 1.3356667915121025e-07, "loss": 0.09, "num_tokens": 1454554086.0, "reward": 0.5831473469734192, "reward_std": 0.10964736575260758, "rewards/accuracy_reward/mean": 0.08928571548312902, "rewards/accuracy_reward/std": 0.2357754185795784, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03752126870676875, "step": 3051 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43080357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 870.7143402099609, "completions/mean_terminated_length": 754.6836090087891, "completions/min_length": 292.25, "completions/min_terminated_length": 292.25, "epoch": 0.9116570831155254, "grad_norm": 0.2462625354528427, "kl": 3.173828125, "learning_rate": 1.3179563562863873e-07, "loss": 0.1506, "num_tokens": 1455018374.0, "reward": 0.5864955633878708, "reward_std": 0.14490696229040623, "rewards/accuracy_reward/mean": 0.10044642840512097, "rewards/accuracy_reward/std": 0.2653613518923521, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.05584815517067909, "step": 3052 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 884.0111999511719, "completions/mean_terminated_length": 736.1494140625, "completions/min_length": 376.5, "completions/min_terminated_length": 376.5, "epoch": 0.9119557912030468, "grad_norm": 0.25975728034973145, "kl": 1.68798828125, "learning_rate": 1.3003633439768182e-07, "loss": 0.0853, "num_tokens": 1455487243.0, "reward": 0.593191996216774, "reward_std": 0.15533526241779327, "rewards/accuracy_reward/mean": 0.10751488339155912, "rewards/accuracy_reward/std": 0.29954819008708, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.03541599866002798, "step": 3053 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.453125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 868.9955749511719, "completions/mean_terminated_length": 741.7597045898438, "completions/min_length": 262.5, "completions/min_terminated_length": 262.5, "epoch": 0.9122544992905683, "grad_norm": 0.22676371037960052, "kl": 1.6328125, "learning_rate": 1.2828877755175163e-07, "loss": 0.0971, "num_tokens": 1455942681.0, "reward": 0.609933078289032, "reward_std": 0.12413921765983105, "rewards/accuracy_reward/mean": 0.11607143143191934, "rewards/accuracy_reward/std": 0.25058707781136036, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 3054 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 869.7344055175781, "completions/mean_terminated_length": 752.6271514892578, "completions/min_length": 385.0, "completions/min_terminated_length": 385.0, "epoch": 0.9125532073780898, "grad_norm": 0.17859835922718048, "kl": 1.833984375, "learning_rate": 1.2655296717028808e-07, "loss": 0.1004, "num_tokens": 1456408178.0, "reward": 0.619419664144516, "reward_std": 0.1283773803152144, "rewards/accuracy_reward/mean": 0.1272321417927742, "rewards/accuracy_reward/std": 0.2769383266568184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 3055 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.30357142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 783.763427734375, "completions/mean_terminated_length": 678.6579895019531, "completions/min_length": 233.25, "completions/min_terminated_length": 233.25, "epoch": 0.9128519154656113, "grad_norm": 0.5508521199226379, "kl": 2.970703125, "learning_rate": 1.2482890531875124e-07, "loss": 0.1618, "num_tokens": 1456835992.0, "reward": 0.6629464626312256, "reward_std": 0.11583795677870512, "rewards/accuracy_reward/mean": 0.1741071417927742, "rewards/accuracy_reward/std": 0.37338782846927643, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05092104524374008, "step": 3056 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3549107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 834.5312957763672, "completions/mean_terminated_length": 733.1521606445312, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "epoch": 0.9131506235531327, "grad_norm": 0.26609909534454346, "kl": 2.408203125, "learning_rate": 1.231165940486234e-07, "loss": 0.1305, "num_tokens": 1457286038.0, "reward": 0.7047991305589676, "reward_std": 0.16854273155331612, "rewards/accuracy_reward/mean": 0.21428570710122585, "rewards/accuracy_reward/std": 0.39643311500549316, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574132680892944, "step": 3057 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 875.9710235595703, "completions/mean_terminated_length": 762.2702178955078, "completions/min_length": 393.25, "completions/min_terminated_length": 393.25, "epoch": 0.9134493316406541, "grad_norm": 0.23198509216308594, "kl": 2.1171875, "learning_rate": 1.2141603539740144e-07, "loss": 0.0963, "num_tokens": 1457754265.0, "reward": 0.5440848469734192, "reward_std": 0.09857119619846344, "rewards/accuracy_reward/mean": 0.05357142840512097, "rewards/accuracy_reward/std": 0.17382354848086834, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.044524834025651217, "step": 3058 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4732142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 879.4933319091797, "completions/mean_terminated_length": 733.6259155273438, "completions/min_length": 283.75, "completions/min_terminated_length": 283.75, "epoch": 0.9137480397281756, "grad_norm": 0.6204888224601746, "kl": 3.591796875, "learning_rate": 1.1972723138860333e-07, "loss": 0.1937, "num_tokens": 1458229830.0, "reward": 0.6612723469734192, "reward_std": 0.1975548416376114, "rewards/accuracy_reward/mean": 0.17857143376022577, "rewards/accuracy_reward/std": 0.3015170991420746, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4827008917927742, "rewards/tag_count_reward/std": 0.06096726004034281, "step": 3059 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2924107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 798.1094207763672, "completions/mean_terminated_length": 705.4988708496094, "completions/min_length": 366.25, "completions/min_terminated_length": 366.25, "epoch": 0.9140467478156971, "grad_norm": 0.4092322289943695, "kl": 2.46484375, "learning_rate": 1.1805018403175383e-07, "loss": 0.1304, "num_tokens": 1458662983.0, "reward": 0.7260044813156128, "reward_std": 0.16661718487739563, "rewards/accuracy_reward/mean": 0.234375, "rewards/accuracy_reward/std": 0.4219934195280075, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 3060 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.5, "completions/mean_length": 809.247802734375, "completions/mean_terminated_length": 711.4049377441406, "completions/min_length": 299.5, "completions/min_terminated_length": 299.5, "epoch": 0.9143454559032186, "grad_norm": 0.32239651679992676, "kl": 1.994140625, "learning_rate": 1.1638489532239339e-07, "loss": 0.12, "num_tokens": 1459100022.0, "reward": 0.7059152126312256, "reward_std": 0.20525386929512024, "rewards/accuracy_reward/mean": 0.21428571734577417, "rewards/accuracy_reward/std": 0.37991267442703247, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.042382154148072004, "step": 3061 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.0, "completions/mean_length": 856.2187957763672, "completions/mean_terminated_length": 769.9785766601562, "completions/min_length": 397.25, "completions/min_terminated_length": 397.25, "epoch": 0.91464416399074, "grad_norm": 0.25900498032569885, "kl": 2.1796875, "learning_rate": 1.1473136724206691e-07, "loss": 0.1174, "num_tokens": 1459560552.0, "reward": 0.7868303954601288, "reward_std": 0.27397168800234795, "rewards/accuracy_reward/mean": 0.2968749925494194, "rewards/accuracy_reward/std": 0.4481728971004486, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04618143197149038, "step": 3062 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 850.6763763427734, "completions/mean_terminated_length": 735.5067138671875, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "epoch": 0.9149428720782615, "grad_norm": 0.4444434642791748, "kl": 1.96875, "learning_rate": 1.1308960175832606e-07, "loss": 0.1033, "num_tokens": 1460016679.0, "reward": 0.6216518133878708, "reward_std": 0.15929418429732323, "rewards/accuracy_reward/mean": 0.12946428637951612, "rewards/accuracy_reward/std": 0.2898252606391907, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 3063 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.328125, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 785.5245819091797, "completions/mean_terminated_length": 670.408203125, "completions/min_length": 320.25, "completions/min_terminated_length": 320.25, "epoch": 0.9152415801657829, "grad_norm": 0.208170548081398, "kl": 2.17578125, "learning_rate": 1.1145960082472928e-07, "loss": 0.1253, "num_tokens": 1460451170.0, "reward": 0.6450893133878708, "reward_std": 0.10781245050020516, "rewards/accuracy_reward/mean": 0.1540178544819355, "rewards/accuracy_reward/std": 0.29205646365880966, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.0447555473074317, "step": 3064 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45535714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 887.9330902099609, "completions/mean_terminated_length": 780.2574920654297, "completions/min_length": 370.5, "completions/min_terminated_length": 370.5, "epoch": 0.9155402882533045, "grad_norm": 0.330433189868927, "kl": 1.61376953125, "learning_rate": 1.0984136638083176e-07, "loss": 0.0903, "num_tokens": 1460925316.0, "reward": 0.6902901977300644, "reward_std": 0.21007918566465378, "rewards/accuracy_reward/mean": 0.1964285741560161, "rewards/accuracy_reward/std": 0.36701036989688873, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.032930306158959866, "step": 3065 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3459821428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.5, "completions/mean_length": 834.7277069091797, "completions/mean_terminated_length": 730.7276153564453, "completions/min_length": 328.5, "completions/min_terminated_length": 328.5, "epoch": 0.9158389963408259, "grad_norm": 0.2559863328933716, "kl": 1.9375, "learning_rate": 1.0823490035218986e-07, "loss": 0.1103, "num_tokens": 1461365962.0, "reward": 0.6601562649011612, "reward_std": 0.14809084543958306, "rewards/accuracy_reward/mean": 0.1674107201397419, "rewards/accuracy_reward/std": 0.3087350130081177, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 3066 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 831.1495971679688, "completions/mean_terminated_length": 717.1528778076172, "completions/min_length": 369.0, "completions/min_terminated_length": 369.0, "epoch": 0.9161377044283474, "grad_norm": 0.26876580715179443, "kl": 1.978515625, "learning_rate": 1.0664020465035785e-07, "loss": 0.115, "num_tokens": 1461811693.0, "reward": 0.7070312649011612, "reward_std": 0.1621302105486393, "rewards/accuracy_reward/mean": 0.2209821455180645, "rewards/accuracy_reward/std": 0.4132017716765404, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04155240673571825, "step": 3067 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4330357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 855.9866485595703, "completions/mean_terminated_length": 748.953857421875, "completions/min_length": 421.75, "completions/min_terminated_length": 421.75, "epoch": 0.9164364125158688, "grad_norm": 0.3240918517112732, "kl": 2.255859375, "learning_rate": 1.0505728117288006e-07, "loss": 0.1151, "num_tokens": 1462261623.0, "reward": 0.7198660969734192, "reward_std": 0.14779706113040447, "rewards/accuracy_reward/mean": 0.2299107164144516, "rewards/accuracy_reward/std": 0.4116643890738487, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 3068 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 853.2768249511719, "completions/mean_terminated_length": 730.8318786621094, "completions/min_length": 363.25, "completions/min_terminated_length": 363.25, "epoch": 0.9167351206033904, "grad_norm": 0.44654035568237305, "kl": 2.115234375, "learning_rate": 1.0348613180329758e-07, "loss": 0.1244, "num_tokens": 1462708579.0, "reward": 0.607700914144516, "reward_std": 0.1630801297724247, "rewards/accuracy_reward/mean": 0.11607142817229033, "rewards/accuracy_reward/std": 0.3134072721004486, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 3069 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43303571428571425, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 859.2902221679688, "completions/mean_terminated_length": 738.9352416992188, "completions/min_length": 310.25, "completions/min_terminated_length": 310.25, "epoch": 0.9170338286909118, "grad_norm": 0.3039184808731079, "kl": 3.0390625, "learning_rate": 1.0192675841113941e-07, "loss": 0.1648, "num_tokens": 1463159461.0, "reward": 0.6618303805589676, "reward_std": 0.19476344622671604, "rewards/accuracy_reward/mean": 0.1763392835855484, "rewards/accuracy_reward/std": 0.3673377148807049, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05782576743513346, "step": 3070 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42633928571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.5, "completions/mean_length": 865.9598693847656, "completions/mean_terminated_length": 750.4240570068359, "completions/min_length": 345.5, "completions/min_terminated_length": 345.5, "epoch": 0.9173325367784333, "grad_norm": 0.23247011005878448, "kl": 1.8671875, "learning_rate": 1.0037916285192129e-07, "loss": 0.0928, "num_tokens": 1463613459.0, "reward": 0.6116071790456772, "reward_std": 0.09163689240813255, "rewards/accuracy_reward/mean": 0.11830357112921774, "rewards/accuracy_reward/std": 0.27925149723887444, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035597205162, "rewards/tag_count_reward/std": 0.03907900024205446, "step": 3071 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.45089285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.75, "completions/mean_length": 880.4866638183594, "completions/mean_terminated_length": 771.9839782714844, "completions/min_length": 448.25, "completions/min_terminated_length": 448.25, "epoch": 0.9176312448659547, "grad_norm": 0.25575342774391174, "kl": 2.287109375, "learning_rate": 9.884334696714459e-08, "loss": 0.1161, "num_tokens": 1464076989.0, "reward": 0.5965402126312256, "reward_std": 0.07360374368727207, "rewards/accuracy_reward/mean": 0.1049107126891613, "rewards/accuracy_reward/std": 0.24074114486575127, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 3072 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 895.9933319091797, "completions/mean_terminated_length": 794.7347717285156, "completions/min_length": 413.5, "completions/min_terminated_length": 413.5, "epoch": 0.9179299529534762, "grad_norm": 0.5256312489509583, "kl": 2.515625, "learning_rate": 9.731931258429638e-08, "loss": 0.1341, "num_tokens": 1464544266.0, "reward": 0.6467634066939354, "reward_std": 0.1311837574467063, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.30145376920700073, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05263457912951708, "step": 3073 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4017857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 854.9129791259766, "completions/mean_terminated_length": 741.4473724365234, "completions/min_length": 311.25, "completions/min_terminated_length": 311.25, "epoch": 0.9182286610409977, "grad_norm": 0.49214473366737366, "kl": 2.912109375, "learning_rate": 9.580706151684271e-08, "loss": 0.1486, "num_tokens": 1465004355.0, "reward": 0.638392873108387, "reward_std": 0.12080336920917034, "rewards/accuracy_reward/mean": 0.1517857126891613, "rewards/accuracy_reward/std": 0.29415491968393326, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.055564895272254944, "step": 3074 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 877.4643249511719, "completions/mean_terminated_length": 738.4584045410156, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.9185273691285192, "grad_norm": 0.24948212504386902, "kl": 2.064453125, "learning_rate": 9.43065955642275e-08, "loss": 0.1117, "num_tokens": 1465470387.0, "reward": 0.6473214626312256, "reward_std": 0.19159581139683723, "rewards/accuracy_reward/mean": 0.15624999720603228, "rewards/accuracy_reward/std": 0.34519270062446594, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.039469920098781586, "step": 3075 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 854.4821929931641, "completions/mean_terminated_length": 719.4688568115234, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.9188260772160406, "grad_norm": 0.19554468989372253, "kl": 1.55078125, "learning_rate": 9.281791651187366e-08, "loss": 0.0892, "num_tokens": 1465914299.0, "reward": 0.7131696939468384, "reward_std": 0.15665392577648163, "rewards/accuracy_reward/mean": 0.21875000279396772, "rewards/accuracy_reward/std": 0.3817545101046562, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196343421936, "rewards/tag_count_reward/std": 0.0369012001901865, "step": 3076 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4397321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 873.1674499511719, "completions/mean_terminated_length": 755.9647216796875, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "epoch": 0.9191247853035621, "grad_norm": 0.26732102036476135, "kl": 1.734375, "learning_rate": 9.134102613117757e-08, "loss": 0.0929, "num_tokens": 1466372390.0, "reward": 0.839285746216774, "reward_std": 0.18652215972542763, "rewards/accuracy_reward/mean": 0.3459821529686451, "rewards/accuracy_reward/std": 0.4509764760732651, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.03973022289574146, "step": 3077 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4575892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 868.8951263427734, "completions/mean_terminated_length": 735.9616546630859, "completions/min_length": 316.5, "completions/min_terminated_length": 316.5, "epoch": 0.9194234933910835, "grad_norm": 0.3060495853424072, "kl": 2.376953125, "learning_rate": 8.987592617950791e-08, "loss": 0.1319, "num_tokens": 1466847495.0, "reward": 0.6227678805589676, "reward_std": 0.16398664563894272, "rewards/accuracy_reward/mean": 0.13392857182770967, "rewards/accuracy_reward/std": 0.32534991949796677, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.05026982259005308, "step": 3078 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3727678571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 844.6317291259766, "completions/mean_terminated_length": 741.046875, "completions/min_length": 373.5, "completions/min_terminated_length": 373.5, "epoch": 0.9197222014786051, "grad_norm": 0.2630522847175598, "kl": 2.0625, "learning_rate": 8.84226184002046e-08, "loss": 0.1098, "num_tokens": 1467299042.0, "reward": 0.6160714477300644, "reward_std": 0.13346346747130156, "rewards/accuracy_reward/mean": 0.12499999930150807, "rewards/accuracy_reward/std": 0.2997561953961849, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.046059842221438885, "step": 3079 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 867.7254791259766, "completions/mean_terminated_length": 750.6697082519531, "completions/min_length": 289.25, "completions/min_terminated_length": 289.25, "epoch": 0.9200209095661265, "grad_norm": 0.4437154531478882, "kl": 2.703125, "learning_rate": 8.698110452257658e-08, "loss": 0.1332, "num_tokens": 1467751207.0, "reward": 0.7047991454601288, "reward_std": 0.17775730416178703, "rewards/accuracy_reward/mean": 0.21651786006987095, "rewards/accuracy_reward/std": 0.36520207300782204, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.05277834925800562, "step": 3080 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5379464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 904.9420013427734, "completions/mean_terminated_length": 769.4500122070312, "completions/min_length": 433.5, "completions/min_terminated_length": 433.5, "epoch": 0.920319617653648, "grad_norm": 0.2800123393535614, "kl": 2.111328125, "learning_rate": 8.555138626189619e-08, "loss": 0.0993, "num_tokens": 1468234269.0, "reward": 0.5457589477300644, "reward_std": 0.10363958403468132, "rewards/accuracy_reward/mean": 0.055803571827709675, "rewards/accuracy_reward/std": 0.21162541955709457, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04908842407166958, "step": 3081 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 825.1696929931641, "completions/mean_terminated_length": 703.7632598876953, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "epoch": 0.9206183257411694, "grad_norm": 0.25241103768348694, "kl": 2.73046875, "learning_rate": 8.413346531940258e-08, "loss": 0.146, "num_tokens": 1468681145.0, "reward": 0.6556920111179352, "reward_std": 0.1614309474825859, "rewards/accuracy_reward/mean": 0.16741071455180645, "rewards/accuracy_reward/std": 0.3716031312942505, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.052778348326683044, "step": 3082 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40848214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 871.7254943847656, "completions/mean_terminated_length": 767.4054412841797, "completions/min_length": 379.25, "completions/min_terminated_length": 379.25, "epoch": 0.920917033828691, "grad_norm": 0.22958703339099884, "kl": 1.841796875, "learning_rate": 8.272734338229727e-08, "loss": 0.0875, "num_tokens": 1469140254.0, "reward": 0.679129496216774, "reward_std": 0.13814156129956245, "rewards/accuracy_reward/mean": 0.18749999860301614, "rewards/accuracy_reward/std": 0.3560684360563755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.045088439248502254, "step": 3083 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.0, "completions/mean_length": 870.4955596923828, "completions/mean_terminated_length": 763.2373199462891, "completions/min_length": 298.25, "completions/min_terminated_length": 298.25, "epoch": 0.9212157419162124, "grad_norm": 0.2766362428665161, "kl": 2.048828125, "learning_rate": 8.133302212373961e-08, "loss": 0.1191, "num_tokens": 1469602284.0, "reward": 0.694754496216774, "reward_std": 0.15068211033940315, "rewards/accuracy_reward/mean": 0.203125, "rewards/accuracy_reward/std": 0.39306437969207764, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04373020678758621, "step": 3084 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 848.091552734375, "completions/mean_terminated_length": 700.2546844482422, "completions/min_length": 337.75, "completions/min_terminated_length": 337.75, "epoch": 0.9215144500037339, "grad_norm": 0.20282629132270813, "kl": 2.333984375, "learning_rate": 7.995050320285025e-08, "loss": 0.1158, "num_tokens": 1470062117.0, "reward": 0.6573661118745804, "reward_std": 0.14451586827635765, "rewards/accuracy_reward/mean": 0.1674107126891613, "rewards/accuracy_reward/std": 0.372220978140831, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04812714271247387, "step": 3085 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.75, "completions/mean_length": 817.5290679931641, "completions/mean_terminated_length": 672.6132965087891, "completions/min_length": 324.0, "completions/min_terminated_length": 324.0, "epoch": 0.9218131580912553, "grad_norm": 0.2670004665851593, "kl": 2.5205078125, "learning_rate": 7.857978826470325e-08, "loss": 0.1334, "num_tokens": 1470498354.0, "reward": 0.6250000298023224, "reward_std": 0.14536517672240734, "rewards/accuracy_reward/mean": 0.1361607126891613, "rewards/accuracy_reward/std": 0.33636731654405594, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.04842412518337369, "step": 3086 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.5, "completions/mean_length": 845.060302734375, "completions/mean_terminated_length": 742.8140258789062, "completions/min_length": 280.25, "completions/min_terminated_length": 280.25, "epoch": 0.9221118661787768, "grad_norm": 0.23434583842754364, "kl": 1.921875, "learning_rate": 7.722087894032948e-08, "loss": 0.0973, "num_tokens": 1470952765.0, "reward": 0.6210937798023224, "reward_std": 0.1602916307747364, "rewards/accuracy_reward/mean": 0.12946428498253226, "rewards/accuracy_reward/std": 0.3170164078474045, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04423765745013952, "step": 3087 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 855.9978179931641, "completions/mean_terminated_length": 752.5072784423828, "completions/min_length": 324.75, "completions/min_terminated_length": 324.75, "epoch": 0.9224105742662982, "grad_norm": 0.3544884920120239, "kl": 2.05859375, "learning_rate": 7.587377684671105e-08, "loss": 0.1149, "num_tokens": 1471402220.0, "reward": 0.6958705633878708, "reward_std": 0.2113368045538664, "rewards/accuracy_reward/mean": 0.20535713993012905, "rewards/accuracy_reward/std": 0.39131055772304535, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983776688576, "step": 3088 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4888392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 896.1339874267578, "completions/mean_terminated_length": 786.9798889160156, "completions/min_length": 359.5, "completions/min_terminated_length": 359.5, "epoch": 0.9227092823538198, "grad_norm": 0.383177787065506, "kl": 1.75537109375, "learning_rate": 7.453848358678018e-08, "loss": 0.0638, "num_tokens": 1471877096.0, "reward": 0.6462053954601288, "reward_std": 0.12576698884367943, "rewards/accuracy_reward/mean": 0.15624999813735485, "rewards/accuracy_reward/std": 0.35425618290901184, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04229862708598375, "step": 3089 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 859.075927734375, "completions/mean_terminated_length": 752.4775390625, "completions/min_length": 262.5, "completions/min_terminated_length": 262.5, "epoch": 0.9230079904413412, "grad_norm": 0.19169504940509796, "kl": 1.9921875, "learning_rate": 7.3215000749417e-08, "loss": 0.1027, "num_tokens": 1472337098.0, "reward": 0.686941996216774, "reward_std": 0.14325003256089985, "rewards/accuracy_reward/mean": 0.1964285708963871, "rewards/accuracy_reward/std": 0.3947790861129761, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04571862844750285, "step": 3090 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 867.1808471679688, "completions/mean_terminated_length": 743.2063446044922, "completions/min_length": 211.25, "completions/min_terminated_length": 211.25, "epoch": 0.9233066985288627, "grad_norm": 0.2923579216003418, "kl": 2.197265625, "learning_rate": 7.19033299094496e-08, "loss": 0.1128, "num_tokens": 1472802347.0, "reward": 0.628348246216774, "reward_std": 0.13785970211029053, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.34400036931037903, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04888886492699385, "step": 3091 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.42410714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.75, "completions/mean_length": 868.0491485595703, "completions/mean_terminated_length": 751.2215118408203, "completions/min_length": 382.25, "completions/min_terminated_length": 382.25, "epoch": 0.9236054066163841, "grad_norm": 0.3019725978374481, "kl": 1.3818359375, "learning_rate": 7.060347262765166e-08, "loss": 0.0767, "num_tokens": 1473266081.0, "reward": 0.8175223618745804, "reward_std": 0.257046464830637, "rewards/accuracy_reward/mean": 0.323660708963871, "rewards/accuracy_reward/std": 0.4590192884206772, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 3092 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 858.9129943847656, "completions/mean_terminated_length": 715.2485198974609, "completions/min_length": 273.5, "completions/min_terminated_length": 273.5, "epoch": 0.9239041147039057, "grad_norm": 0.24324816465377808, "kl": 2.2353515625, "learning_rate": 6.931543045073708e-08, "loss": 0.1077, "num_tokens": 1473723642.0, "reward": 0.6283482536673546, "reward_std": 0.13788539730012417, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.27676524966955185, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 3093 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.234375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 774.1361846923828, "completions/mean_terminated_length": 701.6237182617188, "completions/min_length": 319.5, "completions/min_terminated_length": 319.5, "epoch": 0.9242028227914271, "grad_norm": 0.24983493983745575, "kl": 1.2646484375, "learning_rate": 6.803920491136317e-08, "loss": 0.0757, "num_tokens": 1474147383.0, "reward": 0.7873884290456772, "reward_std": 0.19129342958331108, "rewards/accuracy_reward/mean": 0.2924107164144516, "rewards/accuracy_reward/std": 0.44132163375616074, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776753783226, "rewards/tag_count_reward/std": 0.034184794407337904, "step": 3094 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3995535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.5, "completions/mean_length": 835.0402221679688, "completions/mean_terminated_length": 712.1795043945312, "completions/min_length": 388.75, "completions/min_terminated_length": 388.75, "epoch": 0.9245015308789486, "grad_norm": 0.3164619505405426, "kl": 2.037109375, "learning_rate": 6.677479752812521e-08, "loss": 0.106, "num_tokens": 1474595417.0, "reward": 0.6389509290456772, "reward_std": 0.14786182343959808, "rewards/accuracy_reward/mean": 0.14732142281718552, "rewards/accuracy_reward/std": 0.3157558348029852, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.0448888810351491, "step": 3095 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4196428571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.0, "completions/mean_length": 861.997802734375, "completions/mean_terminated_length": 745.4092559814453, "completions/min_length": 386.25, "completions/min_terminated_length": 386.25, "epoch": 0.92480023896647, "grad_norm": 0.7271896600723267, "kl": 1.4052734375, "learning_rate": 6.552220980555635e-08, "loss": 0.0758, "num_tokens": 1475052408.0, "reward": 0.6724330633878708, "reward_std": 0.10819479450583458, "rewards/accuracy_reward/mean": 0.17857143096625805, "rewards/accuracy_reward/std": 0.35904932022094727, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 3096 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41964285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 862.6004943847656, "completions/mean_terminated_length": 743.2009735107422, "completions/min_length": 336.75, "completions/min_terminated_length": 336.75, "epoch": 0.9250989470539915, "grad_norm": 0.30806204676628113, "kl": 2.146484375, "learning_rate": 6.428144323412544e-08, "loss": 0.1201, "num_tokens": 1475515333.0, "reward": 0.616071455180645, "reward_std": 0.09251549746841192, "rewards/accuracy_reward/mean": 0.125, "rewards/accuracy_reward/std": 0.27998334914445877, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04479066748172045, "step": 3097 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.0, "completions/mean_length": 892.8772735595703, "completions/mean_terminated_length": 782.6439514160156, "completions/min_length": 320.25, "completions/min_terminated_length": 320.25, "epoch": 0.925397655141513, "grad_norm": 0.33313897252082825, "kl": 2.916015625, "learning_rate": 6.305249929023483e-08, "loss": 0.143, "num_tokens": 1475998078.0, "reward": 0.5943080484867096, "reward_std": 0.1414023144170642, "rewards/accuracy_reward/mean": 0.10714285634458065, "rewards/accuracy_reward/std": 0.2537498325109482, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.054124184884130955, "step": 3098 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 823.5826110839844, "completions/mean_terminated_length": 690.9960479736328, "completions/min_length": 326.0, "completions/min_terminated_length": 326.0, "epoch": 0.9256963632290345, "grad_norm": 0.36397847533226013, "kl": 1.59765625, "learning_rate": 6.18353794362192e-08, "loss": 0.0797, "num_tokens": 1476438163.0, "reward": 0.6808036118745804, "reward_std": 0.13640743121504784, "rewards/accuracy_reward/mean": 0.18749999906867743, "rewards/accuracy_reward/std": 0.37073154747486115, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03883600002154708, "step": 3099 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43080357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 865.3571929931641, "completions/mean_terminated_length": 746.1183471679688, "completions/min_length": 364.75, "completions/min_terminated_length": 364.75, "epoch": 0.9259950713165559, "grad_norm": 0.18776819109916687, "kl": 1.69140625, "learning_rate": 6.063008512034452e-08, "loss": 0.075, "num_tokens": 1476890035.0, "reward": 0.707031287252903, "reward_std": 0.14109214209020138, "rewards/accuracy_reward/mean": 0.214285708963871, "rewards/accuracy_reward/std": 0.33996228128671646, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04015073226764798, "step": 3100 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48214285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 863.5937957763672, "completions/mean_terminated_length": 712.0826568603516, "completions/min_length": 288.25, "completions/min_terminated_length": 288.25, "epoch": 0.9262937794040773, "grad_norm": 0.29088279604911804, "kl": 2.095703125, "learning_rate": 5.943661777680354e-08, "loss": 0.121, "num_tokens": 1477349629.0, "reward": 0.6902902126312256, "reward_std": 0.2060207985341549, "rewards/accuracy_reward/mean": 0.2008928544819355, "rewards/accuracy_reward/std": 0.38652077317237854, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.04690983286127448, "step": 3101 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 883.3638763427734, "completions/mean_terminated_length": 769.0437316894531, "completions/min_length": 398.25, "completions/min_terminated_length": 398.25, "epoch": 0.9265924874915988, "grad_norm": 0.2748008668422699, "kl": 1.787109375, "learning_rate": 5.8254978825718065e-08, "loss": 0.0938, "num_tokens": 1477816816.0, "reward": 0.6450893133878708, "reward_std": 0.10613165656104684, "rewards/accuracy_reward/mean": 0.15178571781143546, "rewards/accuracy_reward/std": 0.31198812648653984, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 3102 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.0, "completions/mean_length": 878.5111999511719, "completions/mean_terminated_length": 776.4547882080078, "completions/min_length": 358.0, "completions/min_terminated_length": 358.0, "epoch": 0.9268911955791203, "grad_norm": 0.21080009639263153, "kl": 1.84375, "learning_rate": 5.708516967313338e-08, "loss": 0.1189, "num_tokens": 1478286517.0, "reward": 0.7617187947034836, "reward_std": 0.24172333255410194, "rewards/accuracy_reward/mean": 0.2700892761349678, "rewards/accuracy_reward/std": 0.4151568077504635, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 3103 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.75, "completions/mean_length": 865.3616333007812, "completions/mean_terminated_length": 734.7448120117188, "completions/min_length": 364.5, "completions/min_terminated_length": 364.5, "epoch": 0.9271899036666418, "grad_norm": 0.2618691027164459, "kl": 2.1875, "learning_rate": 5.592719171101935e-08, "loss": 0.1156, "num_tokens": 1478746535.0, "reward": 0.6534598469734192, "reward_std": 0.08637682907283306, "rewards/accuracy_reward/mean": 0.1774553586728871, "rewards/accuracy_reward/std": 0.3377050720155239, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4882812574505806, "rewards/tag_count_reward/std": 0.055484591983258724, "step": 3104 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.25, "completions/mean_length": 806.9553833007812, "completions/mean_terminated_length": 709.9363403320312, "completions/min_length": 314.0, "completions/min_terminated_length": 314.0, "epoch": 0.9274886117541632, "grad_norm": 0.3055482506752014, "kl": 1.9501953125, "learning_rate": 5.4781046317267103e-08, "loss": 0.1023, "num_tokens": 1479175043.0, "reward": 0.6796875298023224, "reward_std": 0.21256758272647858, "rewards/accuracy_reward/mean": 0.18749999813735485, "rewards/accuracy_reward/std": 0.3786954805254936, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.03960080398246646, "step": 3105 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3772321428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.25, "completions/mean_length": 831.8683319091797, "completions/mean_terminated_length": 717.9858551025391, "completions/min_length": 316.75, "completions/min_terminated_length": 316.75, "epoch": 0.9277873198416847, "grad_norm": 0.34808120131492615, "kl": 2.076171875, "learning_rate": 5.364673485568794e-08, "loss": 0.1233, "num_tokens": 1479631032.0, "reward": 0.7154018133878708, "reward_std": 0.18175935372710228, "rewards/accuracy_reward/mean": 0.2232142873108387, "rewards/accuracy_reward/std": 0.4049214571714401, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.04306669719517231, "step": 3106 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3660714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 998.25, "completions/mean_length": 835.2701110839844, "completions/mean_terminated_length": 731.4275817871094, "completions/min_length": 315.25, "completions/min_terminated_length": 315.25, "epoch": 0.9280860279292061, "grad_norm": 0.24048206210136414, "kl": 1.607421875, "learning_rate": 5.252425867601329e-08, "loss": 0.0752, "num_tokens": 1480074129.0, "reward": 0.5652902126312256, "reward_std": 0.10360478609800339, "rewards/accuracy_reward/mean": 0.07142857136204839, "rewards/accuracy_reward/std": 0.19171638786792755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03872338403016329, "step": 3107 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4709821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 880.1495971679688, "completions/mean_terminated_length": 757.1484985351562, "completions/min_length": 464.25, "completions/min_terminated_length": 464.25, "epoch": 0.9283847360167277, "grad_norm": 0.182793989777565, "kl": 1.80078125, "learning_rate": 5.141361911389142e-08, "loss": 0.1041, "num_tokens": 1480538724.0, "reward": 0.6121652126312256, "reward_std": 0.1312539391219616, "rewards/accuracy_reward/mean": 0.11830357508733869, "rewards/accuracy_reward/std": 0.29555412009358406, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616082072258, "rewards/tag_count_reward/std": 0.03732170956209302, "step": 3108 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.75, "completions/mean_length": 842.1607513427734, "completions/mean_terminated_length": 716.9318542480469, "completions/min_length": 360.25, "completions/min_terminated_length": 360.25, "epoch": 0.9286834441042491, "grad_norm": 0.4596833884716034, "kl": 3.31640625, "learning_rate": 5.031481749088296e-08, "loss": 0.1761, "num_tokens": 1480983324.0, "reward": 0.646763414144516, "reward_std": 0.19851798564195633, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.36246488243341446, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491007566452, "rewards/tag_count_reward/std": 0.056990127079188824, "step": 3109 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.49776785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 891.2098541259766, "completions/mean_terminated_length": 760.5640258789062, "completions/min_length": 389.5, "completions/min_terminated_length": 389.5, "epoch": 0.9289821521917706, "grad_norm": 0.24635633826255798, "kl": 2.216796875, "learning_rate": 4.9227855114467595e-08, "loss": 0.112, "num_tokens": 1481457594.0, "reward": 0.6015625298023224, "reward_std": 0.11829814128577709, "rewards/accuracy_reward/mean": 0.11160714388825, "rewards/accuracy_reward/std": 0.269071651622653, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04558529471978545, "step": 3110 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37723214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 816.8460235595703, "completions/mean_terminated_length": 696.0074310302734, "completions/min_length": 288.5, "completions/min_terminated_length": 288.5, "epoch": 0.929280860279292, "grad_norm": 0.23518866300582886, "kl": 2.140625, "learning_rate": 4.815273327803183e-08, "loss": 0.1129, "num_tokens": 1481896709.0, "reward": 0.791294664144516, "reward_std": 0.19262881390750408, "rewards/accuracy_reward/mean": 0.30133928544819355, "rewards/accuracy_reward/std": 0.425493024289608, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.048545535653829575, "step": 3111 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36830357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 828.513427734375, "completions/mean_terminated_length": 719.5969543457031, "completions/min_length": 237.75, "completions/min_terminated_length": 237.75, "epoch": 0.9295795683668135, "grad_norm": 0.266757994890213, "kl": 2.015625, "learning_rate": 4.708945326087677e-08, "loss": 0.1113, "num_tokens": 1482342459.0, "reward": 0.6813616305589676, "reward_std": 0.14315509796142578, "rewards/accuracy_reward/mean": 0.1956845223903656, "rewards/accuracy_reward/std": 0.3721166402101517, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.04378382861614227, "step": 3112 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38392857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 848.0736999511719, "completions/mean_terminated_length": 745.3405151367188, "completions/min_length": 343.5, "completions/min_terminated_length": 343.5, "epoch": 0.929878276454335, "grad_norm": 0.35637226700782776, "kl": 1.8515625, "learning_rate": 4.603801632821148e-08, "loss": 0.0883, "num_tokens": 1482795468.0, "reward": 0.7008928954601288, "reward_std": 0.22257133200764656, "rewards/accuracy_reward/mean": 0.2075892873108387, "rewards/accuracy_reward/std": 0.3997369632124901, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035671710968, "rewards/tag_count_reward/std": 0.04023767542093992, "step": 3113 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.39732142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 859.5736999511719, "completions/mean_terminated_length": 750.7763519287109, "completions/min_length": 370.0, "completions/min_terminated_length": 370.0, "epoch": 0.9301769845418565, "grad_norm": 0.1821703165769577, "kl": 2.3671875, "learning_rate": 4.499842373115404e-08, "loss": 0.1207, "num_tokens": 1483251517.0, "reward": 0.706473246216774, "reward_std": 0.15385880507528782, "rewards/accuracy_reward/mean": 0.21651786006987095, "rewards/accuracy_reward/std": 0.3899834528565407, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553656578064, "rewards/tag_count_reward/std": 0.04903263598680496, "step": 3114 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.46875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 878.8705902099609, "completions/mean_terminated_length": 750.7352447509766, "completions/min_length": 296.25, "completions/min_terminated_length": 296.25, "epoch": 0.9304756926293779, "grad_norm": 0.2171115279197693, "kl": 3.080078125, "learning_rate": 4.397067670672828e-08, "loss": 0.1596, "num_tokens": 1483730019.0, "reward": 0.6082589477300644, "reward_std": 0.1659327670931816, "rewards/accuracy_reward/mean": 0.12276785727590322, "rewards/accuracy_reward/std": 0.31189336627721786, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910597205162, "rewards/tag_count_reward/std": 0.05658396985381842, "step": 3115 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48437499999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 861.9754791259766, "completions/mean_terminated_length": 712.3815460205078, "completions/min_length": 365.5, "completions/min_terminated_length": 365.5, "epoch": 0.9307744007168994, "grad_norm": 0.27486440539360046, "kl": 2.65625, "learning_rate": 4.295477647786039e-08, "loss": 0.1486, "num_tokens": 1484194616.0, "reward": 0.6997768133878708, "reward_std": 0.19073233008384705, "rewards/accuracy_reward/mean": 0.2120535671710968, "rewards/accuracy_reward/std": 0.390978068113327, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05113463895395398, "step": 3116 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 843.3995971679688, "completions/mean_terminated_length": 732.5755310058594, "completions/min_length": 331.5, "completions/min_terminated_length": 331.5, "epoch": 0.9310731088044208, "grad_norm": 0.5145437121391296, "kl": 2.615234375, "learning_rate": 4.195072425338342e-08, "loss": 0.1227, "num_tokens": 1484647163.0, "reward": 0.7248884439468384, "reward_std": 0.13519230228848755, "rewards/accuracy_reward/mean": 0.2366071455180645, "rewards/accuracy_reward/std": 0.41355176270008087, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.048862318973988295, "step": 3117 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4084821428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 853.1161041259766, "completions/mean_terminated_length": 736.540283203125, "completions/min_length": 320.25, "completions/min_terminated_length": 320.25, "epoch": 0.9313718168919424, "grad_norm": 0.3741820752620697, "kl": 1.490234375, "learning_rate": 4.0958521228029454e-08, "loss": 0.0942, "num_tokens": 1485105407.0, "reward": 0.6986607611179352, "reward_std": 0.17732254043221474, "rewards/accuracy_reward/mean": 0.20535714365541935, "rewards/accuracy_reward/std": 0.33595746755599976, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.0371446181088686, "step": 3118 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4040178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 874.1094207763672, "completions/mean_terminated_length": 767.4419250488281, "completions/min_length": 394.75, "completions/min_terminated_length": 394.75, "epoch": 0.9316705249794638, "grad_norm": 0.24916580319404602, "kl": 2.9609375, "learning_rate": 3.997816858243297e-08, "loss": 0.1447, "num_tokens": 1485571152.0, "reward": 0.735491082072258, "reward_std": 0.17687013559043407, "rewards/accuracy_reward/mean": 0.25, "rewards/accuracy_reward/std": 0.4266413301229477, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05698250140994787, "step": 3119 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 833.5714569091797, "completions/mean_terminated_length": 721.7057647705078, "completions/min_length": 337.5, "completions/min_terminated_length": 337.5, "epoch": 0.9319692330669853, "grad_norm": 0.33999910950660706, "kl": 2.2578125, "learning_rate": 3.900966748312862e-08, "loss": 0.1183, "num_tokens": 1486016112.0, "reward": 0.5742187798023224, "reward_std": 0.08420667541213334, "rewards/accuracy_reward/mean": 0.08258928405120969, "rewards/accuracy_reward/std": 0.19835656136274338, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.043475935235619545, "step": 3120 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.515625, "completions/max_length": 1024.0, "completions/max_terminated_length": 999.75, "completions/mean_length": 893.0982360839844, "completions/mean_terminated_length": 751.7427062988281, "completions/min_length": 445.75, "completions/min_terminated_length": 445.75, "epoch": 0.9322679411545067, "grad_norm": 0.32821857929229736, "kl": 1.6982421875, "learning_rate": 3.805301908254455e-08, "loss": 0.1068, "num_tokens": 1486488508.0, "reward": 0.6802455633878708, "reward_std": 0.18240880966186523, "rewards/accuracy_reward/mean": 0.1875, "rewards/accuracy_reward/std": 0.38994649052619934, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818479284644, "step": 3121 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.0, "completions/mean_length": 834.4442291259766, "completions/mean_terminated_length": 706.3563690185547, "completions/min_length": 341.0, "completions/min_terminated_length": 341.0, "epoch": 0.9325666492420283, "grad_norm": 0.28826218843460083, "kl": 1.9912109375, "learning_rate": 3.7108224519010196e-08, "loss": 0.1061, "num_tokens": 1486929651.0, "reward": 0.6623884290456772, "reward_std": 0.17089945450425148, "rewards/accuracy_reward/mean": 0.16964285355061293, "rewards/accuracy_reward/std": 0.35241542756557465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.035161727108061314, "step": 3122 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 875.5312957763672, "completions/mean_terminated_length": 764.2857971191406, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "epoch": 0.9328653573295497, "grad_norm": 0.2913568615913391, "kl": 1.841796875, "learning_rate": 3.617528491674627e-08, "loss": 0.0943, "num_tokens": 1487390529.0, "reward": 0.784598246216774, "reward_std": 0.24240775778889656, "rewards/accuracy_reward/mean": 0.2924107164144516, "rewards/accuracy_reward/std": 0.45457956939935684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.04197291610762477, "step": 3123 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.25, "completions/mean_length": 869.7522735595703, "completions/mean_terminated_length": 743.6023712158203, "completions/min_length": 308.5, "completions/min_terminated_length": 308.5, "epoch": 0.9331640654170712, "grad_norm": 0.31057992577552795, "kl": 2.28515625, "learning_rate": 3.5254201385869215e-08, "loss": 0.1175, "num_tokens": 1487858258.0, "reward": 0.6729910969734192, "reward_std": 0.20646102353930473, "rewards/accuracy_reward/mean": 0.18303571455180645, "rewards/accuracy_reward/std": 0.3801180273294449, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764227330685, "step": 3124 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.0, "completions/mean_length": 860.2991485595703, "completions/mean_terminated_length": 742.6891326904297, "completions/min_length": 261.75, "completions/min_terminated_length": 261.75, "epoch": 0.9334627735045926, "grad_norm": 0.32428061962127686, "kl": 2.1376953125, "learning_rate": 3.4344975022385654e-08, "loss": 0.1112, "num_tokens": 1488316392.0, "reward": 0.6222098469734192, "reward_std": 0.14254706166684628, "rewards/accuracy_reward/mean": 0.13169642933644354, "rewards/accuracy_reward/std": 0.29161653481423855, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04571862844750285, "step": 3125 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4486607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 880.8304138183594, "completions/mean_terminated_length": 760.6255645751953, "completions/min_length": 380.25, "completions/min_terminated_length": 380.25, "epoch": 0.9337614815921141, "grad_norm": 0.2212381809949875, "kl": 1.7333984375, "learning_rate": 3.3447606908196815e-08, "loss": 0.0872, "num_tokens": 1488779980.0, "reward": 0.6489955484867096, "reward_std": 0.10686066490598023, "rewards/accuracy_reward/mean": 0.1584821455180645, "rewards/accuracy_reward/std": 0.3536262921988964, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.03865890856832266, "step": 3126 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.0, "completions/mean_length": 865.3995971679688, "completions/mean_terminated_length": 739.5568695068359, "completions/min_length": 262.25, "completions/min_terminated_length": 262.25, "epoch": 0.9340601896796356, "grad_norm": 0.7220859527587891, "kl": 3.7734375, "learning_rate": 3.256209811108968e-08, "loss": 0.1968, "num_tokens": 1489239263.0, "reward": 0.5931919887661934, "reward_std": 0.15372821874916553, "rewards/accuracy_reward/mean": 0.1138392856810242, "rewards/accuracy_reward/std": 0.27119468711316586, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4793526753783226, "rewards/tag_count_reward/std": 0.06874802615493536, "step": 3127 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 826.5022888183594, "completions/mean_terminated_length": 716.6549530029297, "completions/min_length": 306.25, "completions/min_terminated_length": 306.25, "epoch": 0.9343588977671571, "grad_norm": 0.44545161724090576, "kl": 2.091796875, "learning_rate": 3.168844968474249e-08, "loss": 0.1223, "num_tokens": 1489679184.0, "reward": 0.701450914144516, "reward_std": 0.12564531713724136, "rewards/accuracy_reward/mean": 0.2183779738843441, "rewards/accuracy_reward/std": 0.40894100815057755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 3128 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37499999999999994, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 841.9486999511719, "completions/mean_terminated_length": 739.1022033691406, "completions/min_length": 236.5, "completions/min_terminated_length": 236.5, "epoch": 0.9346576058546785, "grad_norm": 0.491447776556015, "kl": 3.25390625, "learning_rate": 3.082666266872036e-08, "loss": 0.1771, "num_tokens": 1490133977.0, "reward": 0.6668527126312256, "reward_std": 0.20900304056704044, "rewards/accuracy_reward/mean": 0.18080356903374195, "rewards/accuracy_reward/std": 0.37414954602718353, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4860491082072258, "rewards/tag_count_reward/std": 0.056879627518355846, "step": 3129 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1012.25, "completions/mean_length": 858.5870971679688, "completions/mean_terminated_length": 760.7018127441406, "completions/min_length": 395.0, "completions/min_terminated_length": 395.0, "epoch": 0.9349563139422, "grad_norm": 0.19457726180553436, "kl": 1.796875, "learning_rate": 2.9976738088471903e-08, "loss": 0.1031, "num_tokens": 1490587904.0, "reward": 0.713169664144516, "reward_std": 0.1976225096732378, "rewards/accuracy_reward/mean": 0.22098213993012905, "rewards/accuracy_reward/std": 0.3894551992416382, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921874925494194, "rewards/tag_count_reward/std": 0.042415475472807884, "step": 3130 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44196428571428575, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.25, "completions/mean_length": 858.8460235595703, "completions/mean_terminated_length": 734.7717437744141, "completions/min_length": 338.75, "completions/min_terminated_length": 338.75, "epoch": 0.9352550220297214, "grad_norm": 0.2979152500629425, "kl": 3.23046875, "learning_rate": 2.9138676955333676e-08, "loss": 0.1624, "num_tokens": 1491045483.0, "reward": 0.6015625149011612, "reward_std": 0.14127437956631184, "rewards/accuracy_reward/mean": 0.11607142770662904, "rewards/accuracy_reward/std": 0.30048855021595955, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910746216774, "rewards/tag_count_reward/std": 0.05812416970729828, "step": 3131 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.48660714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 876.9375457763672, "completions/mean_terminated_length": 746.6799163818359, "completions/min_length": 408.0, "completions/min_terminated_length": 408.0, "epoch": 0.935553730117243, "grad_norm": 0.2509942948818207, "kl": 2.056640625, "learning_rate": 2.8312480266523556e-08, "loss": 0.1256, "num_tokens": 1491512319.0, "reward": 0.6484375298023224, "reward_std": 0.1674376241862774, "rewards/accuracy_reward/mean": 0.15848213993012905, "rewards/accuracy_reward/std": 0.35548459738492966, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 3132 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41294642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 855.3527069091797, "completions/mean_terminated_length": 739.1562042236328, "completions/min_length": 279.25, "completions/min_terminated_length": 279.25, "epoch": 0.9358524382047644, "grad_norm": 0.3525622487068176, "kl": 1.419921875, "learning_rate": 2.7498149005144025e-08, "loss": 0.0792, "num_tokens": 1491961469.0, "reward": 0.7260044813156128, "reward_std": 0.1552425567060709, "rewards/accuracy_reward/mean": 0.2321428544819355, "rewards/accuracy_reward/std": 0.41381795704364777, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.03841549064964056, "step": 3133 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4464285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 870.7388763427734, "completions/mean_terminated_length": 747.8258361816406, "completions/min_length": 437.75, "completions/min_terminated_length": 437.75, "epoch": 0.9361511462922859, "grad_norm": 0.4569578170776367, "kl": 3.25, "learning_rate": 2.6695684140175537e-08, "loss": 0.1805, "num_tokens": 1492432232.0, "reward": 0.5630580559372902, "reward_std": 0.14958205539733171, "rewards/accuracy_reward/mean": 0.07812499930150807, "rewards/accuracy_reward/std": 0.20106449350714684, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4849330335855484, "rewards/tag_count_reward/std": 0.05936532001942396, "step": 3134 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4107142857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 850.4442291259766, "completions/mean_terminated_length": 729.0558776855469, "completions/min_length": 330.0, "completions/min_terminated_length": 330.0, "epoch": 0.9364498543798073, "grad_norm": 0.23916186392307281, "kl": 1.70703125, "learning_rate": 2.5905086626480947e-08, "loss": 0.1114, "num_tokens": 1492887887.0, "reward": 0.6997768133878708, "reward_std": 0.2081635296344757, "rewards/accuracy_reward/mean": 0.2075892873108387, "rewards/accuracy_reward/std": 0.3903351426124573, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 3135 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3816964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.0, "completions/mean_length": 823.2098541259766, "completions/mean_terminated_length": 700.5821685791016, "completions/min_length": 333.0, "completions/min_terminated_length": 333.0, "epoch": 0.9367485624673288, "grad_norm": 0.2350703626871109, "kl": 2.384765625, "learning_rate": 2.512635740480218e-08, "loss": 0.1457, "num_tokens": 1493333517.0, "reward": 0.7589286118745804, "reward_std": 0.1794657576829195, "rewards/accuracy_reward/mean": 0.2678571417927742, "rewards/accuracy_reward/std": 0.44029345363378525, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04589572083204985, "step": 3136 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.38839285714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1005.5, "completions/mean_length": 821.7678833007812, "completions/mean_terminated_length": 707.9750213623047, "completions/min_length": 369.5, "completions/min_terminated_length": 369.5, "epoch": 0.9370472705548503, "grad_norm": 0.23537923395633698, "kl": 1.875, "learning_rate": 2.4359497401758026e-08, "loss": 0.1045, "num_tokens": 1493770597.0, "reward": 0.6082589626312256, "reward_std": 0.1576485992409289, "rewards/accuracy_reward/mean": 0.11607142770662904, "rewards/accuracy_reward/std": 0.24239300563931465, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 3137 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.75, "completions/mean_length": 841.6964721679688, "completions/mean_terminated_length": 709.8034973144531, "completions/min_length": 367.5, "completions/min_terminated_length": 367.5, "epoch": 0.9373459786423718, "grad_norm": 0.2378091961145401, "kl": 3.201171875, "learning_rate": 2.3604507529843e-08, "loss": 0.1594, "num_tokens": 1494236733.0, "reward": 0.5758928805589676, "reward_std": 0.15666988864541054, "rewards/accuracy_reward/mean": 0.09151785774156451, "rewards/accuracy_reward/std": 0.26809509843587875, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.484375, "rewards/tag_count_reward/std": 0.05966775305569172, "step": 3138 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.36607142857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.75, "completions/mean_length": 833.4754791259766, "completions/mean_terminated_length": 724.3377532958984, "completions/min_length": 261.75, "completions/min_terminated_length": 261.75, "epoch": 0.9376446867298932, "grad_norm": 0.5328736305236816, "kl": 1.87109375, "learning_rate": 2.2861388687430708e-08, "loss": 0.1209, "num_tokens": 1494687970.0, "reward": 0.6601562798023224, "reward_std": 0.12568321451544762, "rewards/accuracy_reward/mean": 0.16741071362048388, "rewards/accuracy_reward/std": 0.3503861203789711, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.042059858329594135, "step": 3139 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4642857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.5, "completions/mean_length": 895.6428985595703, "completions/mean_terminated_length": 789.7792358398438, "completions/min_length": 324.75, "completions/min_terminated_length": 324.75, "epoch": 0.9379433948174147, "grad_norm": 0.40081310272216797, "kl": 2.435546875, "learning_rate": 2.2130141758764933e-08, "loss": 0.1193, "num_tokens": 1495156946.0, "reward": 0.5295759066939354, "reward_std": 0.10042706038802862, "rewards/accuracy_reward/mean": 0.04241071571595967, "rewards/accuracy_reward/std": 0.1599063239991665, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.05512027069926262, "step": 3140 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4598214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.75, "completions/mean_length": 868.1585388183594, "completions/mean_terminated_length": 740.0345306396484, "completions/min_length": 352.25, "completions/min_terminated_length": 352.25, "epoch": 0.9382421029049361, "grad_norm": 0.38670819997787476, "kl": 2.556640625, "learning_rate": 2.1410767613965212e-08, "loss": 0.1356, "num_tokens": 1495614937.0, "reward": 0.6121652126312256, "reward_std": 0.16154029965400696, "rewards/accuracy_reward/mean": 0.12611607182770967, "rewards/accuracy_reward/std": 0.3155512660741806, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810389876366, "step": 3141 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 840.4397735595703, "completions/mean_terminated_length": 732.7913055419922, "completions/min_length": 294.75, "completions/min_terminated_length": 294.75, "epoch": 0.9385408109924577, "grad_norm": 0.1957172304391861, "kl": 2.40234375, "learning_rate": 2.0703267109023483e-08, "loss": 0.1223, "num_tokens": 1496061070.0, "reward": 0.6484375149011612, "reward_std": 0.17179395258426666, "rewards/accuracy_reward/mean": 0.16406249813735485, "rewards/accuracy_reward/std": 0.3593614846467972, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04778412822633982, "step": 3142 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 907.1228179931641, "completions/mean_terminated_length": 785.9500579833984, "completions/min_length": 379.25, "completions/min_terminated_length": 379.25, "epoch": 0.9388395190799791, "grad_norm": 0.23957805335521698, "kl": 2.0732421875, "learning_rate": 2.0007641085803e-08, "loss": 0.1139, "num_tokens": 1496542725.0, "reward": 0.6529018133878708, "reward_std": 0.15697935968637466, "rewards/accuracy_reward/mean": 0.1607142835855484, "rewards/accuracy_reward/std": 0.3580983802676201, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.041829145047813654, "step": 3143 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 837.8303985595703, "completions/mean_terminated_length": 734.8813171386719, "completions/min_length": 286.25, "completions/min_terminated_length": 286.25, "epoch": 0.9391382271675005, "grad_norm": 0.22341932356357574, "kl": 1.9453125, "learning_rate": 1.93238903720383e-08, "loss": 0.0929, "num_tokens": 1496990137.0, "reward": 0.6406250298023224, "reward_std": 0.13665154110640287, "rewards/accuracy_reward/mean": 0.14955356856808066, "rewards/accuracy_reward/std": 0.3191557005047798, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04479066748172045, "step": 3144 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.47767857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 875.810302734375, "completions/mean_terminated_length": 741.1834411621094, "completions/min_length": 261.5, "completions/min_terminated_length": 261.5, "epoch": 0.939436935255022, "grad_norm": 0.24475127458572388, "kl": 2.4765625, "learning_rate": 1.86520157813308e-08, "loss": 0.138, "num_tokens": 1497464916.0, "reward": 0.6456473469734192, "reward_std": 0.1720675677061081, "rewards/accuracy_reward/mean": 0.15624999906867743, "rewards/accuracy_reward/std": 0.34866364300251007, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.0494418740272522, "step": 3145 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.31026785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 796.138427734375, "completions/mean_terminated_length": 698.2706604003906, "completions/min_length": 321.25, "completions/min_terminated_length": 321.25, "epoch": 0.9397356433425434, "grad_norm": 0.49313032627105713, "kl": 2.345703125, "learning_rate": 1.79920181131521e-08, "loss": 0.1468, "num_tokens": 1497901010.0, "reward": 0.7053571790456772, "reward_std": 0.13560869544744492, "rewards/accuracy_reward/mean": 0.214285708963871, "rewards/accuracy_reward/std": 0.39476320147514343, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.04640317242592573, "step": 3146 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4508928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.0, "completions/mean_length": 875.6161041259766, "completions/mean_terminated_length": 756.7171325683594, "completions/min_length": 374.0, "completions/min_terminated_length": 374.0, "epoch": 0.940034351430065, "grad_norm": 0.23998932540416718, "kl": 2.630859375, "learning_rate": 1.7343898152841765e-08, "loss": 0.1326, "num_tokens": 1498367158.0, "reward": 0.6981027126312256, "reward_std": 0.18638317473232746, "rewards/accuracy_reward/mean": 0.2098214328289032, "rewards/accuracy_reward/std": 0.40574541687965393, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918293237686, "step": 3147 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3035714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 800.3795013427734, "completions/mean_terminated_length": 705.9864349365234, "completions/min_length": 268.75, "completions/min_terminated_length": 268.75, "epoch": 0.9403330595175864, "grad_norm": 0.3348669707775116, "kl": 2.53515625, "learning_rate": 1.6707656671604012e-08, "loss": 0.1265, "num_tokens": 1498798704.0, "reward": 0.6149553805589676, "reward_std": 0.17181003838777542, "rewards/accuracy_reward/mean": 0.1272321455180645, "rewards/accuracy_reward/std": 0.3291653022170067, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232238650322, "rewards/tag_count_reward/std": 0.051208219956606627, "step": 3148 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.37276785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.25, "completions/mean_length": 862.3080902099609, "completions/mean_terminated_length": 769.1147766113281, "completions/min_length": 406.0, "completions/min_terminated_length": 406.0, "epoch": 0.9406317676051079, "grad_norm": 0.3226349949836731, "kl": 2.00390625, "learning_rate": 1.608329442651213e-08, "loss": 0.1226, "num_tokens": 1499255178.0, "reward": 0.7154017984867096, "reward_std": 0.1978142037987709, "rewards/accuracy_reward/mean": 0.22321428917348385, "rewards/accuracy_reward/std": 0.3985918089747429, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.043066698126494884, "step": 3149 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.25, "completions/mean_length": 811.7254791259766, "completions/mean_terminated_length": 692.9615478515625, "completions/min_length": 269.5, "completions/min_terminated_length": 269.5, "epoch": 0.9409304756926293, "grad_norm": 0.48867669701576233, "kl": 2.55078125, "learning_rate": 1.5470812160499614e-08, "loss": 0.1576, "num_tokens": 1499695247.0, "reward": 0.7862723469734192, "reward_std": 0.2070738859474659, "rewards/accuracy_reward/mean": 0.2968749925494194, "rewards/accuracy_reward/std": 0.45052453875541687, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.047245155554264784, "step": 3150 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4308035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 845.7165374755859, "completions/mean_terminated_length": 712.4507751464844, "completions/min_length": 354.75, "completions/min_terminated_length": 354.75, "epoch": 0.9412291837801509, "grad_norm": 0.2367599606513977, "kl": 2.548828125, "learning_rate": 1.487021060236904e-08, "loss": 0.1463, "num_tokens": 1500147312.0, "reward": 0.6685268133878708, "reward_std": 0.15164186106994748, "rewards/accuracy_reward/mean": 0.1785714291036129, "rewards/accuracy_reward/std": 0.29576531052589417, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04812714271247387, "step": 3151 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4241071428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 865.9888916015625, "completions/mean_terminated_length": 759.6003265380859, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.9415278918676723, "grad_norm": 0.25944381952285767, "kl": 2.20703125, "learning_rate": 1.4281490466780956e-08, "loss": 0.1054, "num_tokens": 1500611323.0, "reward": 0.5926339477300644, "reward_std": 0.11786693939939141, "rewards/accuracy_reward/mean": 0.10825892956927419, "rewards/accuracy_reward/std": 0.292818870395422, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764227330685, "step": 3152 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3683035714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 867.8192291259766, "completions/mean_terminated_length": 781.5353393554688, "completions/min_length": 421.75, "completions/min_terminated_length": 421.75, "epoch": 0.9418265999551938, "grad_norm": 0.3186301290988922, "kl": 2.353515625, "learning_rate": 1.370465245426167e-08, "loss": 0.1166, "num_tokens": 1501089114.0, "reward": 0.599888414144516, "reward_std": 0.15145190432667732, "rewards/accuracy_reward/mean": 0.10937499720603228, "rewards/accuracy_reward/std": 0.3080361634492874, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04757413361221552, "step": 3153 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 866.763427734375, "completions/mean_terminated_length": 773.2829742431641, "completions/min_length": 375.25, "completions/min_terminated_length": 375.25, "epoch": 0.9421253080427152, "grad_norm": 0.2204885184764862, "kl": 1.9453125, "learning_rate": 1.313969725119657e-08, "loss": 0.1058, "num_tokens": 1501550368.0, "reward": 0.681919664144516, "reward_std": 0.17071537487208843, "rewards/accuracy_reward/mean": 0.1919642873108387, "rewards/accuracy_reward/std": 0.39166110008955, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 3154 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5223214285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 902.1562957763672, "completions/mean_terminated_length": 771.1439056396484, "completions/min_length": 336.25, "completions/min_terminated_length": 336.25, "epoch": 0.9424240161302367, "grad_norm": 0.4504075348377228, "kl": 2.37109375, "learning_rate": 1.2586625529832363e-08, "loss": 0.1332, "num_tokens": 1502032678.0, "reward": 0.659598246216774, "reward_std": 0.17169984290376306, "rewards/accuracy_reward/mean": 0.171875, "rewards/accuracy_reward/std": 0.31273650377988815, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05083381850272417, "step": 3155 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4799107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.75, "completions/mean_length": 884.6607513427734, "completions/mean_terminated_length": 756.0560455322266, "completions/min_length": 364.25, "completions/min_terminated_length": 364.25, "epoch": 0.9427227242177582, "grad_norm": 0.19284315407276154, "kl": 2.2421875, "learning_rate": 1.2045437948275952e-08, "loss": 0.1064, "num_tokens": 1502504814.0, "reward": 0.6674107387661934, "reward_std": 0.1568289641290903, "rewards/accuracy_reward/mean": 0.17857143003493547, "rewards/accuracy_reward/std": 0.28269394487142563, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 3156 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.75, "completions/mean_length": 859.4710083007812, "completions/mean_terminated_length": 737.3667602539062, "completions/min_length": 355.0, "completions/min_terminated_length": 355.0, "epoch": 0.9430214323052797, "grad_norm": 0.24875372648239136, "kl": 2.61328125, "learning_rate": 1.1516135150493323e-08, "loss": 0.1418, "num_tokens": 1502964801.0, "reward": 0.6450893133878708, "reward_std": 0.1866868920624256, "rewards/accuracy_reward/mean": 0.15625000186264515, "rewards/accuracy_reward/std": 0.3535183444619179, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.05171788763254881, "step": 3157 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4665178571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 883.3995971679688, "completions/mean_terminated_length": 770.8811340332031, "completions/min_length": 404.5, "completions/min_terminated_length": 404.5, "epoch": 0.9433201403928011, "grad_norm": 0.35132896900177, "kl": 2.40234375, "learning_rate": 1.0998717766307343e-08, "loss": 0.1351, "num_tokens": 1503427572.0, "reward": 0.5915178805589676, "reward_std": 0.10926171578466892, "rewards/accuracy_reward/mean": 0.10267857206054032, "rewards/accuracy_reward/std": 0.27857106551527977, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4888392835855484, "rewards/tag_count_reward/std": 0.049996999092400074, "step": 3158 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3325892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 805.8839569091797, "completions/mean_terminated_length": 696.7494506835938, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "epoch": 0.9436188484803226, "grad_norm": 0.26427245140075684, "kl": 2.720703125, "learning_rate": 1.0493186411398848e-08, "loss": 0.1482, "num_tokens": 1503859824.0, "reward": 0.7405134215950966, "reward_std": 0.16438185423612595, "rewards/accuracy_reward/mean": 0.2522321380674839, "rewards/accuracy_reward/std": 0.35080110281705856, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.0513117304071784, "step": 3159 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428572, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 851.2745819091797, "completions/mean_terminated_length": 746.1083984375, "completions/min_length": 344.75, "completions/min_terminated_length": 344.75, "epoch": 0.943917556567844, "grad_norm": 0.27769285440444946, "kl": 2.65625, "learning_rate": 9.999541687306657e-09, "loss": 0.1489, "num_tokens": 1504312459.0, "reward": 0.5524553805589676, "reward_std": 0.12090621562674642, "rewards/accuracy_reward/mean": 0.06473214295692742, "rewards/accuracy_reward/std": 0.18746157549321651, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 3160 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 849.6875457763672, "completions/mean_terminated_length": 753.2246246337891, "completions/min_length": 410.75, "completions/min_terminated_length": 410.75, "epoch": 0.9442162646553656, "grad_norm": 0.6874380111694336, "kl": 2.541015625, "learning_rate": 9.517784181422018e-09, "loss": 0.1462, "num_tokens": 1504759183.0, "reward": 0.7248884290456772, "reward_std": 0.18143458478152752, "rewards/accuracy_reward/mean": 0.2366071455180645, "rewards/accuracy_reward/std": 0.4164591580629349, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05158455390483141, "step": 3161 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4174107142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.5, "completions/mean_length": 863.6295013427734, "completions/mean_terminated_length": 749.1610870361328, "completions/min_length": 367.75, "completions/min_terminated_length": 367.75, "epoch": 0.944514972742887, "grad_norm": 0.36589962244033813, "kl": 1.84375, "learning_rate": 9.047914466996377e-09, "loss": 0.0986, "num_tokens": 1505211753.0, "reward": 0.7209821790456772, "reward_std": 0.19604645296931267, "rewards/accuracy_reward/mean": 0.2299107126891613, "rewards/accuracy_reward/std": 0.4061521664261818, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714328289032, "rewards/tag_count_reward/std": 0.046403173357248306, "step": 3162 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 854.4308319091797, "completions/mean_terminated_length": 756.9373779296875, "completions/min_length": 381.5, "completions/min_terminated_length": 381.5, "epoch": 0.9448136808304085, "grad_norm": 0.24531789124011993, "kl": 1.6171875, "learning_rate": 8.589933103132498e-09, "loss": 0.0921, "num_tokens": 1505671354.0, "reward": 0.6177455484867096, "reward_std": 0.09810198098421097, "rewards/accuracy_reward/mean": 0.1261160708963871, "rewards/accuracy_reward/std": 0.27233588695526123, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4949776828289032, "rewards/tag_count_reward/std": 0.03359846491366625, "step": 3163 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3794642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1006.5, "completions/mean_length": 823.2076110839844, "completions/mean_terminated_length": 700.3470916748047, "completions/min_length": 304.5, "completions/min_terminated_length": 304.5, "epoch": 0.9451123889179299, "grad_norm": 0.22408653795719147, "kl": 2.19921875, "learning_rate": 8.143840634786682e-09, "loss": 0.1289, "num_tokens": 1506108759.0, "reward": 0.6824777126312256, "reward_std": 0.196101576089859, "rewards/accuracy_reward/mean": 0.19196428591385484, "rewards/accuracy_reward/std": 0.35768166556954384, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.04626983776688576, "step": 3164 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4263392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.0, "completions/mean_length": 846.6428985595703, "completions/mean_terminated_length": 717.0053558349609, "completions/min_length": 325.0, "completions/min_terminated_length": 325.0, "epoch": 0.9454110970054515, "grad_norm": 0.22355058789253235, "kl": 2.048828125, "learning_rate": 7.70963759277099e-09, "loss": 0.1046, "num_tokens": 1506561319.0, "reward": 0.6350446790456772, "reward_std": 0.1331392340362072, "rewards/accuracy_reward/mean": 0.14508928544819355, "rewards/accuracy_reward/std": 0.3408683277666569, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.048127141781151295, "step": 3165 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5267857142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 901.1875305175781, "completions/mean_terminated_length": 772.8378295898438, "completions/min_length": 361.0, "completions/min_terminated_length": 361.0, "epoch": 0.9457098050929729, "grad_norm": 0.27670612931251526, "kl": 2.751953125, "learning_rate": 7.2873244937476935e-09, "loss": 0.1238, "num_tokens": 1507039867.0, "reward": 0.6250000298023224, "reward_std": 0.12588175013661385, "rewards/accuracy_reward/mean": 0.1383928582072258, "rewards/accuracy_reward/std": 0.29212769120931625, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4866071417927742, "rewards/tag_count_reward/std": 0.054607308469712734, "step": 3166 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.5200892857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1013.75, "completions/mean_length": 913.6451416015625, "completions/mean_terminated_length": 797.3340301513672, "completions/min_length": 362.0, "completions/min_terminated_length": 362.0, "epoch": 0.9460085131804944, "grad_norm": 0.3092593848705292, "kl": 2.337890625, "learning_rate": 6.876901840231487e-09, "loss": 0.1178, "num_tokens": 1507527772.0, "reward": 0.5820312798023224, "reward_std": 0.11466831341385841, "rewards/accuracy_reward/mean": 0.09375000186264515, "rewards/accuracy_reward/std": 0.26242388039827347, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.48828125, "rewards/tag_count_reward/std": 0.05181918200105429, "step": 3167 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1004.75, "completions/mean_length": 874.6964721679688, "completions/mean_terminated_length": 771.2102355957031, "completions/min_length": 306.0, "completions/min_terminated_length": 306.0, "epoch": 0.9463072212680158, "grad_norm": 1.4323543310165405, "kl": 2.287109375, "learning_rate": 6.478370120591715e-09, "loss": 0.1209, "num_tokens": 1508000564.0, "reward": 0.5920758992433548, "reward_std": 0.1445693988353014, "rewards/accuracy_reward/mean": 0.10267857136204839, "rewards/accuracy_reward/std": 0.27933038026094437, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973246216774, "rewards/tag_count_reward/std": 0.050403155386447906, "step": 3168 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3839285714285714, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 830.5692443847656, "completions/mean_terminated_length": 716.4116668701172, "completions/min_length": 408.75, "completions/min_terminated_length": 408.75, "epoch": 0.9466059293555373, "grad_norm": 0.4901524484157562, "kl": 1.5615234375, "learning_rate": 6.091729809042379e-09, "loss": 0.0955, "num_tokens": 1508445459.0, "reward": 0.7209821790456772, "reward_std": 0.19400377944111824, "rewards/accuracy_reward/mean": 0.22767856903374195, "rewards/accuracy_reward/std": 0.3910066857933998, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4933035746216774, "rewards/tag_count_reward/std": 0.03475248906761408, "step": 3169 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.5, "completions/mean_length": 853.0536193847656, "completions/mean_terminated_length": 740.3832855224609, "completions/min_length": 374.5, "completions/min_terminated_length": 374.5, "epoch": 0.9469046374430587, "grad_norm": 0.48507651686668396, "kl": 2.236328125, "learning_rate": 5.716981365654351e-09, "loss": 0.1156, "num_tokens": 1508901515.0, "reward": 0.6194196864962578, "reward_std": 0.11154067609459162, "rewards/accuracy_reward/mean": 0.12946428917348385, "rewards/accuracy_reward/std": 0.26332438737154007, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04823764320462942, "step": 3170 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.5, "completions/mean_length": 869.3861846923828, "completions/mean_terminated_length": 761.4715728759766, "completions/min_length": 294.75, "completions/min_terminated_length": 294.75, "epoch": 0.9472033455305803, "grad_norm": 0.4032002091407776, "kl": 2.107421875, "learning_rate": 5.354125236343155e-09, "loss": 0.1076, "num_tokens": 1509359512.0, "reward": 0.6462053805589676, "reward_std": 0.09247427061200142, "rewards/accuracy_reward/mean": 0.1562500037252903, "rewards/accuracy_reward/std": 0.3027029260993004, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04809202253818512, "step": 3171 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.75, "completions/mean_length": 840.3080749511719, "completions/mean_terminated_length": 724.9104919433594, "completions/min_length": 380.75, "completions/min_terminated_length": 380.75, "epoch": 0.9475020536181017, "grad_norm": 0.27696850895881653, "kl": 1.89453125, "learning_rate": 5.003161852876748e-09, "loss": 0.1018, "num_tokens": 1509802722.0, "reward": 0.7215402275323868, "reward_std": 0.2252776250243187, "rewards/accuracy_reward/mean": 0.2299107126891613, "rewards/accuracy_reward/std": 0.40997108817100525, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.042347033973783255, "step": 3172 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.2611607142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1002.25, "completions/mean_length": 777.5736846923828, "completions/mean_terminated_length": 690.7471618652344, "completions/min_length": 338.75, "completions/min_terminated_length": 338.75, "epoch": 0.9478007617056232, "grad_norm": 0.3890869915485382, "kl": 2.455078125, "learning_rate": 4.6640916328710705e-09, "loss": 0.1611, "num_tokens": 1510227043.0, "reward": 0.7220982611179352, "reward_std": 0.19862723164260387, "rewards/accuracy_reward/mean": 0.2321428582072258, "rewards/accuracy_reward/std": 0.3996850252151489, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553582072258, "rewards/tag_count_reward/std": 0.04843503516167402, "step": 3173 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.75, "completions/mean_length": 876.2879791259766, "completions/mean_terminated_length": 762.5367431640625, "completions/min_length": 395.75, "completions/min_terminated_length": 395.75, "epoch": 0.9480994697931446, "grad_norm": 0.21595068275928497, "kl": 2.099609375, "learning_rate": 4.336914979787832e-09, "loss": 0.1102, "num_tokens": 1510696836.0, "reward": 0.667410746216774, "reward_std": 0.20770669728517532, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.3501499630510807, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.045552390627563, "step": 3174 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.359375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 827.7053985595703, "completions/mean_terminated_length": 722.43603515625, "completions/min_length": 342.75, "completions/min_terminated_length": 342.75, "epoch": 0.9483981778806662, "grad_norm": 0.22466887533664703, "kl": 2.41015625, "learning_rate": 4.021632282938947e-09, "loss": 0.1212, "num_tokens": 1511140768.0, "reward": 0.6774553880095482, "reward_std": 0.14273477904498577, "rewards/accuracy_reward/mean": 0.18973213923163712, "rewards/accuracy_reward/std": 0.348630640655756, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05333347339183092, "step": 3175 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1021.0, "completions/mean_length": 869.1942443847656, "completions/mean_terminated_length": 763.9550933837891, "completions/min_length": 346.25, "completions/min_terminated_length": 346.25, "epoch": 0.9486968859681876, "grad_norm": 0.34692999720573425, "kl": 1.830078125, "learning_rate": 3.7182439174832106e-09, "loss": 0.119, "num_tokens": 1511611879.0, "reward": 0.6690848469734192, "reward_std": 0.2104444019496441, "rewards/accuracy_reward/mean": 0.1763392873108387, "rewards/accuracy_reward/std": 0.37924277782440186, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.492745541036129, "rewards/tag_count_reward/std": 0.04065818386152387, "step": 3176 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.44642857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 993.25, "completions/mean_length": 865.3906402587891, "completions/mean_terminated_length": 740.0570220947266, "completions/min_length": 342.75, "completions/min_terminated_length": 342.75, "epoch": 0.9489955940557091, "grad_norm": 0.2520141005516052, "kl": 2.3359375, "learning_rate": 3.4267502444274013e-09, "loss": 0.1369, "num_tokens": 1512077174.0, "reward": 0.7154018133878708, "reward_std": 0.18738732486963272, "rewards/accuracy_reward/mean": 0.22544642724096775, "rewards/accuracy_reward/std": 0.40819472819566727, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4899553507566452, "rewards/tag_count_reward/std": 0.04778381250798702, "step": 3177 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.25, "completions/mean_length": 870.9531707763672, "completions/mean_terminated_length": 765.5382385253906, "completions/min_length": 350.25, "completions/min_terminated_length": 350.25, "epoch": 0.9492943021432305, "grad_norm": 0.2439442276954651, "kl": 1.8671875, "learning_rate": 3.1471516106207355e-09, "loss": 0.1022, "num_tokens": 1512543425.0, "reward": 0.6311384290456772, "reward_std": 0.15939549542963505, "rewards/accuracy_reward/mean": 0.1383928544819355, "rewards/accuracy_reward/std": 0.29203853011131287, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.04175196494907141, "step": 3178 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.34375, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 820.4732513427734, "completions/mean_terminated_length": 714.45458984375, "completions/min_length": 398.75, "completions/min_terminated_length": 398.75, "epoch": 0.949593010230752, "grad_norm": 0.3882460296154022, "kl": 1.3466796875, "learning_rate": 2.879448348762637e-09, "loss": 0.0673, "num_tokens": 1512983893.0, "reward": 0.6796875298023224, "reward_std": 0.09221185091882944, "rewards/accuracy_reward/mean": 0.1830357126891613, "rewards/accuracy_reward/std": 0.37442974746227264, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4966517835855484, "rewards/tag_count_reward/std": 0.02435629488900304, "step": 3179 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.35044642857142855, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 829.0580749511719, "completions/mean_terminated_length": 731.1716156005859, "completions/min_length": 354.5, "completions/min_terminated_length": 354.5, "epoch": 0.9498917183182735, "grad_norm": 0.43039408326148987, "kl": 1.958984375, "learning_rate": 2.6236407773960747e-09, "loss": 0.1213, "num_tokens": 1513418063.0, "reward": 0.612723246216774, "reward_std": 0.13933849707245827, "rewards/accuracy_reward/mean": 0.12053571292199194, "rewards/accuracy_reward/std": 0.2679440211504698, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875074505806, "rewards/tag_count_reward/std": 0.042172474320977926, "step": 3180 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3705357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 834.9777069091797, "completions/mean_terminated_length": 726.2464904785156, "completions/min_length": 330.5, "completions/min_terminated_length": 330.5, "epoch": 0.950190426405795, "grad_norm": 0.45394790172576904, "kl": 2.681640625, "learning_rate": 2.379729200908676e-09, "loss": 0.1241, "num_tokens": 1513865125.0, "reward": 0.632254496216774, "reward_std": 0.16958051174879074, "rewards/accuracy_reward/mean": 0.1450892835855484, "rewards/accuracy_reward/std": 0.31912095472216606, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651828289032, "rewards/tag_count_reward/std": 0.05492102913558483, "step": 3181 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3080357142857143, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.0, "completions/mean_length": 824.8192138671875, "completions/mean_terminated_length": 736.3091125488281, "completions/min_length": 313.5, "completions/min_terminated_length": 313.5, "epoch": 0.9504891344933164, "grad_norm": 0.23808881640434265, "kl": 2.1953125, "learning_rate": 2.147713909534943e-09, "loss": 0.1195, "num_tokens": 1514298212.0, "reward": 0.6378348469734192, "reward_std": 0.13403314165771008, "rewards/accuracy_reward/mean": 0.1473214253783226, "rewards/accuracy_reward/std": 0.3375255689024925, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.04771790374070406, "step": 3182 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3928571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1020.25, "completions/mean_length": 856.3281555175781, "completions/mean_terminated_length": 749.9064483642578, "completions/min_length": 295.5, "completions/min_terminated_length": 295.5, "epoch": 0.9507878425808379, "grad_norm": 0.4380035698413849, "kl": 1.431640625, "learning_rate": 1.9275951793518154e-09, "loss": 0.0691, "num_tokens": 1514764695.0, "reward": 0.633928582072258, "reward_std": 0.11872266046702862, "rewards/accuracy_reward/mean": 0.1383928586728871, "rewards/accuracy_reward/std": 0.3092210702598095, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4955357164144516, "rewards/tag_count_reward/std": 0.03177628107368946, "step": 3183 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43080357142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.75, "completions/mean_length": 832.7098541259766, "completions/mean_terminated_length": 694.9183654785156, "completions/min_length": 279.5, "completions/min_terminated_length": 279.5, "epoch": 0.9510865506683593, "grad_norm": 0.3362296223640442, "kl": 2.783203125, "learning_rate": 1.7193732722808886e-09, "loss": 0.1596, "num_tokens": 1515209205.0, "reward": 0.6696428805589676, "reward_std": 0.20460142567753792, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.38380543142557144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.488839291036129, "rewards/tag_count_reward/std": 0.04890321707352996, "step": 3184 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3638392857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.25, "completions/mean_length": 834.7232513427734, "completions/mean_terminated_length": 728.6054229736328, "completions/min_length": 357.5, "completions/min_terminated_length": 357.5, "epoch": 0.9513852587558809, "grad_norm": 0.2943936288356781, "kl": 2.216796875, "learning_rate": 1.5230484360873043e-09, "loss": 0.1341, "num_tokens": 1515656905.0, "reward": 0.6724330633878708, "reward_std": 0.17463091760873795, "rewards/accuracy_reward/mean": 0.1808035746216774, "rewards/accuracy_reward/std": 0.38469983637332916, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04438142944127321, "step": 3185 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.3883928571428571, "completions/max_length": 1024.0, "completions/max_terminated_length": 1009.5, "completions/mean_length": 865.841552734375, "completions/mean_terminated_length": 770.7032928466797, "completions/min_length": 347.25, "completions/min_terminated_length": 347.25, "epoch": 0.9516839668434023, "grad_norm": 0.3016451299190521, "kl": 1.927734375, "learning_rate": 1.3386209043819708e-09, "loss": 0.0977, "num_tokens": 1516120034.0, "reward": 0.6227678954601288, "reward_std": 0.15475591644644737, "rewards/accuracy_reward/mean": 0.1316964291036129, "rewards/accuracy_reward/std": 0.2853784039616585, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04529811907559633, "step": 3186 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1008.25, "completions/mean_length": 870.5469207763672, "completions/mean_terminated_length": 750.2992706298828, "completions/min_length": 414.5, "completions/min_terminated_length": 414.5, "epoch": 0.9519826749309237, "grad_norm": 0.17819765210151672, "kl": 1.609375, "learning_rate": 1.1660908966171224e-09, "loss": 0.0759, "num_tokens": 1516584007.0, "reward": 0.6501116454601288, "reward_std": 0.12813865207135677, "rewards/accuracy_reward/mean": 0.15624999906867743, "rewards/accuracy_reward/std": 0.35099051147699356, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4938616007566452, "rewards/tag_count_reward/std": 0.037908039055764675, "step": 3187 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4620535714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1007.5, "completions/mean_length": 878.2969207763672, "completions/mean_terminated_length": 753.5159759521484, "completions/min_length": 328.25, "completions/min_terminated_length": 328.25, "epoch": 0.9522813830184452, "grad_norm": 0.42243823409080505, "kl": 2.248046875, "learning_rate": 1.0054586180863191e-09, "loss": 0.1216, "num_tokens": 1517053452.0, "reward": 0.714285746216774, "reward_std": 0.22379763051867485, "rewards/accuracy_reward/mean": 0.2232142835855484, "rewards/accuracy_reward/std": 0.4127473309636116, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4910714253783226, "rewards/tag_count_reward/std": 0.04444765392690897, "step": 3188 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 859.154052734375, "completions/mean_terminated_length": 740.1422424316406, "completions/min_length": 344.0, "completions/min_terminated_length": 344.0, "epoch": 0.9525800911059666, "grad_norm": 0.25295737385749817, "kl": 2.328125, "learning_rate": 8.567242599299974e-10, "loss": 0.1315, "num_tokens": 1517518833.0, "reward": 0.670200914144516, "reward_std": 0.1354390699416399, "rewards/accuracy_reward/mean": 0.1808035708963871, "rewards/accuracy_reward/std": 0.3773120045661926, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4893973171710968, "rewards/tag_count_reward/std": 0.04929810296744108, "step": 3189 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4129464285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1011.75, "completions/mean_length": 851.3281555175781, "completions/mean_terminated_length": 731.1662445068359, "completions/min_length": 352.5, "completions/min_terminated_length": 352.5, "epoch": 0.9528787991934882, "grad_norm": 0.2923806309700012, "kl": 3.015625, "learning_rate": 7.198879991276996e-10, "loss": 0.1624, "num_tokens": 1517972932.0, "reward": 0.666294664144516, "reward_std": 0.1505499854683876, "rewards/accuracy_reward/mean": 0.1808035671710968, "rewards/accuracy_reward/std": 0.36950117349624634, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.057093001902103424, "step": 3190 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.40625, "completions/max_length": 1024.0, "completions/max_terminated_length": 1010.5, "completions/mean_length": 859.5803985595703, "completions/mean_terminated_length": 749.5712738037109, "completions/min_length": 308.75, "completions/min_terminated_length": 308.75, "epoch": 0.9531775072810096, "grad_norm": 0.18368804454803467, "kl": 1.7119140625, "learning_rate": 5.949499985025142e-10, "loss": 0.0896, "num_tokens": 1518433432.0, "reward": 0.6043527126312256, "reward_std": 0.10438340716063976, "rewards/accuracy_reward/mean": 0.11681547435000539, "rewards/accuracy_reward/std": 0.28585001081228256, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4927455335855484, "rewards/tag_count_reward/std": 0.040006961207836866, "step": 3191 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43973214285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.0, "completions/mean_length": 862.2723541259766, "completions/mean_terminated_length": 742.4342498779297, "completions/min_length": 346.25, "completions/min_terminated_length": 346.25, "epoch": 0.9534762153685311, "grad_norm": 0.39620545506477356, "kl": 2.671875, "learning_rate": 4.819104067199653e-10, "loss": 0.1482, "num_tokens": 1518892498.0, "reward": 0.757254496216774, "reward_std": 0.24310190975666046, "rewards/accuracy_reward/mean": 0.2700892873108387, "rewards/accuracy_reward/std": 0.43724311143159866, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4871651753783226, "rewards/tag_count_reward/std": 0.054050604812800884, "step": 3192 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.43526785714285715, "completions/max_length": 1024.0, "completions/max_terminated_length": 1014.25, "completions/mean_length": 858.5044860839844, "completions/mean_terminated_length": 737.1644287109375, "completions/min_length": 369.75, "completions/min_terminated_length": 369.75, "epoch": 0.9537749234560525, "grad_norm": 0.22974181175231934, "kl": 1.78515625, "learning_rate": 3.807693582869032e-10, "loss": 0.0857, "num_tokens": 1519352116.0, "reward": 0.5703125149011612, "reward_std": 0.09350762702524662, "rewards/accuracy_reward/mean": 0.07886904734186828, "rewards/accuracy_reward/std": 0.23226883448660374, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4921875, "rewards/tag_count_reward/std": 0.042559245601296425, "step": 3193 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4419642857142857, "completions/max_length": 1024.0, "completions/max_terminated_length": 1019.5, "completions/mean_length": 868.1674499511719, "completions/mean_terminated_length": 759.2219543457031, "completions/min_length": 333.75, "completions/min_terminated_length": 333.75, "epoch": 0.954073631543574, "grad_norm": 0.3414892852306366, "kl": 2.310546875, "learning_rate": 2.9152697355261383e-10, "loss": 0.1315, "num_tokens": 1519812319.0, "reward": 0.6333705633878708, "reward_std": 0.1789878960698843, "rewards/accuracy_reward/mean": 0.1428571417927742, "rewards/accuracy_reward/std": 0.34738200157880783, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133992433548, "rewards/tag_count_reward/std": 0.047917463816702366, "step": 3194 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4441964285714286, "completions/max_length": 1024.0, "completions/max_terminated_length": 1016.5, "completions/mean_length": 858.9620819091797, "completions/mean_terminated_length": 731.2533569335938, "completions/min_length": 313.0, "completions/min_terminated_length": 313.0, "epoch": 0.9543723396310955, "grad_norm": 0.34687772393226624, "kl": 3.236328125, "learning_rate": 2.1418335870770912e-10, "loss": 0.1913, "num_tokens": 1520266270.0, "reward": 0.6395089477300644, "reward_std": 0.19248921424150467, "rewards/accuracy_reward/mean": 0.1540178582072258, "rewards/accuracy_reward/std": 0.3612808287143707, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4854910671710968, "rewards/tag_count_reward/std": 0.05674629285931587, "step": 3195 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4285714285714285, "completions/max_length": 1024.0, "completions/max_terminated_length": 1017.75, "completions/mean_length": 871.9486999511719, "completions/mean_terminated_length": 759.3917999267578, "completions/min_length": 247.0, "completions/min_terminated_length": 247.0, "epoch": 0.954671047718617, "grad_norm": 0.19879966974258423, "kl": 2.021484375, "learning_rate": 1.487386057841267e-10, "loss": 0.0916, "num_tokens": 1520732183.0, "reward": 0.5864955633878708, "reward_std": 0.1546299159526825, "rewards/accuracy_reward/mean": 0.0959821417927742, "rewards/accuracy_reward/std": 0.245246272534132, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4905133917927742, "rewards/tag_count_reward/std": 0.047574134543538094, "step": 3196 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571429, "completions/max_length": 1024.0, "completions/max_terminated_length": 1015.5, "completions/mean_length": 867.3527069091797, "completions/mean_terminated_length": 742.7713928222656, "completions/min_length": 266.0, "completions/min_terminated_length": 266.0, "epoch": 0.9549697558061384, "grad_norm": 0.25280579924583435, "kl": 1.96875, "learning_rate": 9.519279265512993e-11, "loss": 0.0975, "num_tokens": 1521194085.0, "reward": 0.672433078289032, "reward_std": 0.15229077264666557, "rewards/accuracy_reward/mean": 0.18080356903374195, "rewards/accuracy_reward/std": 0.30282870680093765, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.491629458963871, "rewards/tag_count_reward/std": 0.0442376583814621, "step": 3197 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.4553571428571428, "completions/max_length": 1024.0, "completions/max_terminated_length": 1018.25, "completions/mean_length": 897.0670013427734, "completions/mean_terminated_length": 791.0603942871094, "completions/min_length": 423.75, "completions/min_terminated_length": 423.75, "epoch": 0.9552684638936599, "grad_norm": 0.21159473061561584, "kl": 1.541015625, "learning_rate": 5.3545983035308004e-11, "loss": 0.0782, "num_tokens": 1521667347.0, "reward": 0.6171875298023224, "reward_std": 0.06677421508356929, "rewards/accuracy_reward/mean": 0.1313244067132473, "rewards/accuracy_reward/std": 0.2830173522233963, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4944196417927742, "rewards/tag_count_reward/std": 0.03600697731599212, "step": 3198 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.41517857142857145, "completions/max_length": 1024.0, "completions/max_terminated_length": 1003.0, "completions/mean_length": 838.200927734375, "completions/mean_terminated_length": 703.3813781738281, "completions/min_length": 289.25, "completions/min_terminated_length": 289.25, "epoch": 0.9555671719811814, "grad_norm": 0.24910224974155426, "kl": 1.990234375, "learning_rate": 2.379822648168606e-11, "loss": 0.1227, "num_tokens": 1522113357.0, "reward": 0.6612723469734192, "reward_std": 0.1671992652118206, "rewards/accuracy_reward/mean": 0.1696428544819355, "rewards/accuracy_reward/std": 0.36885181814432144, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4916294664144516, "rewards/tag_count_reward/std": 0.04458098765462637, "step": 3199 }, { "clip_ratio": 0.0, "completions/clipped_ratio": 0.421875, "completions/max_length": 1024.0, "completions/max_terminated_length": 1022.25, "completions/mean_length": 874.8995971679688, "completions/mean_terminated_length": 771.0624542236328, "completions/min_length": 312.75, "completions/min_terminated_length": 312.75, "epoch": 0.9558658800687029, "grad_norm": 0.1828087568283081, "kl": 2.38671875, "learning_rate": 5.949558390394572e-12, "loss": 0.1028, "num_tokens": 1522575568.0, "reward": 0.5747768133878708, "reward_std": 0.11844344530254602, "rewards/accuracy_reward/mean": 0.08705357136204839, "rewards/accuracy_reward/std": 0.21549613401293755, "rewards/format_reward/mean": 0.0, "rewards/format_reward/std": 0.0, "rewards/tag_count_reward/mean": 0.4877232164144516, "rewards/tag_count_reward/std": 0.05315246619284153, "step": 3200 }, { "epoch": 0.9558658800687029, "step": 3200, "total_flos": 0.0, "train_loss": 0.148394811640417, "train_runtime": 218075.7377, "train_samples_per_second": 6.574, "train_steps_per_second": 0.015 } ], "logging_steps": 1, "max_steps": 3200, "num_input_tokens_seen": 1522575568, "num_train_epochs": 1, "save_steps": 800.0, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 16, "trial_name": null, "trial_params": null }