{ "best_global_step": null, "best_metric": null, "best_model_checkpoint": null, "epoch": 0.5434782608695652, "eval_steps": 500, "global_step": 200, "is_hyper_param_search": false, "is_local_process_zero": true, "is_world_process_zero": true, "log_history": [ { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.41999998688697815, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1671.2799072265625, "completions/mean_terminated_length": 1398.4827880859375, "completions/min_length": 992.0, "completions/min_terminated_length": 992.0, "entropy": 0.21248915791511536, "epoch": 0.002717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01557081751525402, "learning_rate": 0.0, "loss": 0.08887895941734314, "num_tokens": 86304.0, "reward": -0.21605467796325684, "reward_std": 0.656523585319519, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8160547018051147, "rewards/length_penalty/std": 0.18964654207229614, "sampling/importance_sampling_ratio/max": 1.5222556591033936, "sampling/importance_sampling_ratio/mean": 0.9927070140838623, "sampling/importance_sampling_ratio/min": 0.6435883045196533, "sampling/sampling_logp_difference/max": 0.44069600105285645, "sampling/sampling_logp_difference/mean": 0.013988969847559929, "step": 1, "step_time": 27.487226913915947 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.6800000071525574, "completions/max_length": 2048.0, "completions/max_terminated_length": 1991.0, "completions/mean_length": 1941.47998046875, "completions/mean_terminated_length": 1715.125, "completions/min_length": 1079.0, "completions/min_terminated_length": 1079.0, "entropy": 0.2905632793903351, "epoch": 0.005434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.016905339434742928, "learning_rate": 5e-06, "loss": 0.07884831726551056, "num_tokens": 186618.0, "reward": -0.7279882431030273, "reward_std": 0.4917677640914917, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.9479882717132568, "rewards/length_penalty/std": 0.10518474876880646, "sampling/importance_sampling_ratio/max": 1.450933814048767, "sampling/importance_sampling_ratio/mean": 0.9901063442230225, "sampling/importance_sampling_ratio/min": 0.3688630163669586, "sampling/sampling_logp_difference/max": 0.9973299503326416, "sampling/sampling_logp_difference/mean": 0.01780563034117222, "step": 2, "step_time": 29.082292816136032 }, { "clip_ratio/high_max": 0.0007578980759717524, "clip_ratio/high_mean": 0.0007578980759717524, "clip_ratio/low_mean": 0.0001570210457430221, "clip_ratio/low_min": 0.0001570210457430221, "clip_ratio/region_mean": 0.000914919120259583, "completions/clipped_ratio": 0.29999998211860657, "completions/max_length": 2048.0, "completions/max_terminated_length": 1981.0, "completions/mean_length": 1629.5799560546875, "completions/mean_terminated_length": 1450.2572021484375, "completions/min_length": 792.0, "completions/min_terminated_length": 792.0, "entropy": 0.26428356766700745, "epoch": 0.008152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.016904909163713455, "learning_rate": 1e-05, "loss": 0.13087651133537292, "num_tokens": 270367.0, "reward": -0.07569335401058197, "reward_std": 0.5990626215934753, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.7956933379173279, "rewards/length_penalty/std": 0.19082719087600708, "sampling/importance_sampling_ratio/max": 1.6641552448272705, "sampling/importance_sampling_ratio/mean": 0.9910364151000977, "sampling/importance_sampling_ratio/min": 0.5466911196708679, "sampling/sampling_logp_difference/max": 0.6038713455200195, "sampling/sampling_logp_difference/mean": 0.016635609790682793, "step": 3, "step_time": 27.224814475979656 }, { "clip_ratio/high_max": 0.000611801867489703, "clip_ratio/high_mean": 0.000611801867489703, "clip_ratio/low_mean": 0.00013670422049472108, "clip_ratio/low_min": 0.00013670422049472108, "clip_ratio/region_mean": 0.0007485060661565513, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2040.0, "completions/mean_length": 1611.1400146484375, "completions/mean_terminated_length": 1343.3870849609375, "completions/min_length": 743.0, "completions/min_terminated_length": 743.0, "entropy": 0.266324457526207, "epoch": 0.010869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017118919640779495, "learning_rate": 1.5e-05, "loss": 0.12693746387958527, "num_tokens": 353824.0, "reward": -0.12668944895267487, "reward_std": 0.65364009141922, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851815819740295, "rewards/length_penalty/mean": -0.7866894602775574, "rewards/length_penalty/std": 0.22417497634887695, "sampling/importance_sampling_ratio/max": 1.4671045541763306, "sampling/importance_sampling_ratio/mean": 0.990697979927063, "sampling/importance_sampling_ratio/min": 0.5993587374687195, "sampling/sampling_logp_difference/max": 0.511894941329956, "sampling/sampling_logp_difference/mean": 0.016942981630563736, "step": 4, "step_time": 27.82042264076881 }, { "clip_ratio/high_max": 0.00018956458079628647, "clip_ratio/high_mean": 0.00018956458079628647, "clip_ratio/low_mean": 0.00011366580656613223, "clip_ratio/low_min": 0.00011366580656613223, "clip_ratio/region_mean": 0.00030323039391078055, "completions/clipped_ratio": 0.7599999904632568, "completions/max_length": 2048.0, "completions/max_terminated_length": 1969.0, "completions/mean_length": 1931.8399658203125, "completions/mean_terminated_length": 1564.0, "completions/min_length": 1090.0, "completions/min_terminated_length": 1090.0, "entropy": 0.28429176211357116, "epoch": 0.01358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.01620331034064293, "learning_rate": 2e-05, "loss": 0.0873071625828743, "num_tokens": 454126.0, "reward": -0.7032812237739563, "reward_std": 0.5363532304763794, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.9432812333106995, "rewards/length_penalty/std": 0.11653515696525574, "sampling/importance_sampling_ratio/max": 1.466483473777771, "sampling/importance_sampling_ratio/mean": 0.9900121688842773, "sampling/importance_sampling_ratio/min": 0.6077903509140015, "sampling/sampling_logp_difference/max": 0.4979252815246582, "sampling/sampling_logp_difference/mean": 0.01787548139691353, "step": 5, "step_time": 29.146478729089722 }, { "clip_ratio/high_max": 0.0009160175977740437, "clip_ratio/high_mean": 0.0009160175977740437, "clip_ratio/low_mean": 0.00011906316940439865, "clip_ratio/low_min": 0.00011906316940439865, "clip_ratio/region_mean": 0.0010350807569921017, "completions/clipped_ratio": 0.5999999642372131, "completions/max_length": 2048.0, "completions/max_terminated_length": 1882.0, "completions/mean_length": 1688.4599609375, "completions/mean_terminated_length": 1149.1500244140625, "completions/min_length": 791.0, "completions/min_terminated_length": 791.0, "entropy": 0.2593019545078278, "epoch": 0.016304347826086956, "frac_reward_zero_std": 0.0, "grad_norm": 0.018829405307769775, "learning_rate": 2.5e-05, "loss": 0.09051939845085144, "num_tokens": 541539.0, "reward": -0.4044433534145355, "reward_std": 0.7161739468574524, "rewards/correctness/mean": 0.41999998688697815, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8244433403015137, "rewards/length_penalty/std": 0.23776446282863617, "sampling/importance_sampling_ratio/max": 1.4669702053070068, "sampling/importance_sampling_ratio/mean": 0.9909037351608276, "sampling/importance_sampling_ratio/min": 0.6560537219047546, "sampling/sampling_logp_difference/max": 0.4215126037597656, "sampling/sampling_logp_difference/mean": 0.01694655790925026, "step": 6, "step_time": 27.49979746970348 }, { "clip_ratio/high_max": 0.0005782971245935186, "clip_ratio/high_mean": 0.0005782971245935186, "clip_ratio/low_mean": 0.00015609066904289647, "clip_ratio/low_min": 0.00015609066904289647, "clip_ratio/region_mean": 0.0007343877688981593, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 1964.0, "completions/mean_length": 1656.7999267578125, "completions/mean_terminated_length": 1349.4285888671875, "completions/min_length": 822.0, "completions/min_terminated_length": 822.0, "entropy": 0.25222333073616027, "epoch": 0.019021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.017305413261055946, "learning_rate": 3e-05, "loss": 0.13140623271465302, "num_tokens": 626609.0, "reward": -0.208984375, "reward_std": 0.6656795144081116, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.8089843988418579, "rewards/length_penalty/std": 0.2056698501110077, "sampling/importance_sampling_ratio/max": 1.8690052032470703, "sampling/importance_sampling_ratio/mean": 0.9913385510444641, "sampling/importance_sampling_ratio/min": 0.2781076431274414, "sampling/sampling_logp_difference/max": 1.2797470092773438, "sampling/sampling_logp_difference/mean": 0.016055544838309288, "step": 7, "step_time": 27.746631886344403 }, { "clip_ratio/high_max": 0.0003524738865962718, "clip_ratio/high_mean": 0.0003524738865962718, "clip_ratio/low_mean": 0.00013881978520657867, "clip_ratio/low_min": 0.00013881978520657867, "clip_ratio/region_mean": 0.0004912936768960207, "completions/clipped_ratio": 0.5399999618530273, "completions/max_length": 2048.0, "completions/max_terminated_length": 1927.0, "completions/mean_length": 1758.419921875, "completions/mean_terminated_length": 1418.478271484375, "completions/min_length": 829.0, "completions/min_terminated_length": 829.0, "entropy": 0.21002306342124938, "epoch": 0.021739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.01688527688384056, "learning_rate": 3.5e-05, "loss": 0.10030547529459, "num_tokens": 716680.0, "reward": -0.3786035180091858, "reward_std": 0.6632672548294067, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.8586035370826721, "rewards/length_penalty/std": 0.18755297362804413, "sampling/importance_sampling_ratio/max": 1.512620449066162, "sampling/importance_sampling_ratio/mean": 0.9925761818885803, "sampling/importance_sampling_ratio/min": 0.6414684653282166, "sampling/sampling_logp_difference/max": 0.44399523735046387, "sampling/sampling_logp_difference/mean": 0.013780373148620129, "step": 8, "step_time": 27.67905417783186 }, { "clip_ratio/high_max": 0.0006790827785152942, "clip_ratio/high_mean": 0.0006790827785152942, "clip_ratio/low_mean": 0.00013338743592612446, "clip_ratio/low_min": 0.00013338743592612446, "clip_ratio/region_mean": 0.0008124702260829508, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1630.8199462890625, "completions/mean_terminated_length": 1513.15380859375, "completions/min_length": 877.0, "completions/min_terminated_length": 877.0, "entropy": 0.23098267912864684, "epoch": 0.024456521739130436, "frac_reward_zero_std": 0.0, "grad_norm": 0.017829010263085365, "learning_rate": 4e-05, "loss": 0.10412289947271347, "num_tokens": 800691.0, "reward": -0.19629882276058197, "reward_std": 0.5814719796180725, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7962988018989563, "rewards/length_penalty/std": 0.1725773811340332, "sampling/importance_sampling_ratio/max": 1.4684697389602661, "sampling/importance_sampling_ratio/mean": 0.992029070854187, "sampling/importance_sampling_ratio/min": 0.5849054455757141, "sampling/sampling_logp_difference/max": 0.5363050699234009, "sampling/sampling_logp_difference/mean": 0.015129257924854755, "step": 9, "step_time": 27.676380766090006 }, { "clip_ratio/high_max": 0.0006911004064022563, "clip_ratio/high_mean": 0.0006911004064022563, "clip_ratio/low_mean": 0.00017352561262669043, "clip_ratio/low_min": 0.00017352561262669043, "clip_ratio/region_mean": 0.0008646260248497128, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1973.0, "completions/mean_length": 1501.8199462890625, "completions/mean_terminated_length": 1347.769287109375, "completions/min_length": 653.0, "completions/min_terminated_length": 653.0, "entropy": 0.2444721668958664, "epoch": 0.02717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.017643559724092484, "learning_rate": 4.5e-05, "loss": 0.15564076602458954, "num_tokens": 880602.0, "reward": -0.07331054657697678, "reward_std": 0.640110194683075, "rewards/correctness/mean": 0.6600000262260437, "rewards/correctness/std": 0.47851815819740295, "rewards/length_penalty/mean": -0.7333105206489563, "rewards/length_penalty/std": 0.2063654214143753, "sampling/importance_sampling_ratio/max": 1.4597691297531128, "sampling/importance_sampling_ratio/mean": 0.991223156452179, "sampling/importance_sampling_ratio/min": 0.6512584090232849, "sampling/sampling_logp_difference/max": 0.4288487434387207, "sampling/sampling_logp_difference/mean": 0.015730587765574455, "step": 10, "step_time": 27.221213835058734 }, { "clip_ratio/high_max": 0.0006887678755447268, "clip_ratio/high_mean": 0.0006887678755447268, "clip_ratio/low_mean": 0.0002177180489525199, "clip_ratio/low_min": 0.0002177180489525199, "clip_ratio/region_mean": 0.000906485936138779, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 2045.0, "completions/mean_length": 1849.159912109375, "completions/mean_terminated_length": 1679.77783203125, "completions/min_length": 912.0, "completions/min_terminated_length": 912.0, "entropy": 0.24389304220676422, "epoch": 0.029891304347826088, "frac_reward_zero_std": 0.0, "grad_norm": 0.017000572755932808, "learning_rate": 5e-05, "loss": 0.11899248510599136, "num_tokens": 975480.0, "reward": -0.34291014075279236, "reward_std": 0.5967607498168945, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.9029101729393005, "rewards/length_penalty/std": 0.13233329355716705, "sampling/importance_sampling_ratio/max": 1.4651082754135132, "sampling/importance_sampling_ratio/mean": 0.9913073182106018, "sampling/importance_sampling_ratio/min": 0.5848537087440491, "sampling/sampling_logp_difference/max": 0.5363935232162476, "sampling/sampling_logp_difference/mean": 0.01577482745051384, "step": 11, "step_time": 28.25258613587357 }, { "clip_ratio/high_max": 0.0004739030671771616, "clip_ratio/high_mean": 0.0004739030671771616, "clip_ratio/low_mean": 0.00015672652443754487, "clip_ratio/low_min": 0.00015672652443754487, "clip_ratio/region_mean": 0.0006306295865215362, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2017.0, "completions/mean_length": 1769.02001953125, "completions/mean_terminated_length": 1549.821533203125, "completions/min_length": 979.0, "completions/min_terminated_length": 979.0, "entropy": 0.2830862611532211, "epoch": 0.03260869565217391, "frac_reward_zero_std": 0.0, "grad_norm": 0.018326712772250175, "learning_rate": 5e-05, "loss": 0.1064625158905983, "num_tokens": 1066571.0, "reward": -0.2837792932987213, "reward_std": 0.6255027055740356, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.4985693693161011, "rewards/length_penalty/mean": -0.8637793064117432, "rewards/length_penalty/std": 0.16100184619426727, "sampling/importance_sampling_ratio/max": 1.4403636455535889, "sampling/importance_sampling_ratio/mean": 0.9901345372200012, "sampling/importance_sampling_ratio/min": 0.653232753276825, "sampling/sampling_logp_difference/max": 0.42582178115844727, "sampling/sampling_logp_difference/mean": 0.01810324937105179, "step": 12, "step_time": 28.36113432282582 }, { "clip_ratio/high_max": 0.0005078502028482035, "clip_ratio/high_mean": 0.0005078502028482035, "clip_ratio/low_mean": 0.0001291289911023341, "clip_ratio/low_min": 0.0001291289911023341, "clip_ratio/region_mean": 0.000636979192495346, "completions/clipped_ratio": 0.3400000035762787, "completions/max_length": 2048.0, "completions/max_terminated_length": 1870.0, "completions/mean_length": 1617.5399169921875, "completions/mean_terminated_length": 1395.7879638671875, "completions/min_length": 873.0, "completions/min_terminated_length": 873.0, "entropy": 0.25736612975597384, "epoch": 0.035326086956521736, "frac_reward_zero_std": 0.0, "grad_norm": 0.01772652007639408, "learning_rate": 5e-05, "loss": 0.13986119627952576, "num_tokens": 1150468.0, "reward": -0.26981446146965027, "reward_std": 0.6443559527397156, "rewards/correctness/mean": 0.5199999809265137, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.7898144721984863, "rewards/length_penalty/std": 0.18370111286640167, "sampling/importance_sampling_ratio/max": 1.4670993089675903, "sampling/importance_sampling_ratio/mean": 0.9913054704666138, "sampling/importance_sampling_ratio/min": 0.5635415315628052, "sampling/sampling_logp_difference/max": 0.5735142230987549, "sampling/sampling_logp_difference/mean": 0.01675332337617874, "step": 13, "step_time": 27.25929268822074 }, { "clip_ratio/high_max": 0.0005579496617428958, "clip_ratio/high_mean": 0.0005579496617428958, "clip_ratio/low_mean": 9.234682802343741e-05, "clip_ratio/low_min": 9.234682802343741e-05, "clip_ratio/region_mean": 0.0006502964824903756, "completions/clipped_ratio": 0.5199999809265137, "completions/max_length": 2048.0, "completions/max_terminated_length": 1780.0, "completions/mean_length": 1718.39990234375, "completions/mean_terminated_length": 1361.3333740234375, "completions/min_length": 929.0, "completions/min_terminated_length": 929.0, "entropy": 0.2600396126508713, "epoch": 0.03804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.016946176066994667, "learning_rate": 5e-05, "loss": 0.13000647723674774, "num_tokens": 1238948.0, "reward": -0.3590624928474426, "reward_std": 0.6806654930114746, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.839062511920929, "rewards/length_penalty/std": 0.1944756656885147, "sampling/importance_sampling_ratio/max": 1.4597631692886353, "sampling/importance_sampling_ratio/mean": 0.9908787608146667, "sampling/importance_sampling_ratio/min": 0.6390702128410339, "sampling/sampling_logp_difference/max": 0.4477410316467285, "sampling/sampling_logp_difference/mean": 0.01679503731429577, "step": 14, "step_time": 28.23892678786069 }, { "clip_ratio/high_max": 0.0004477548645809293, "clip_ratio/high_mean": 0.0004477548645809293, "clip_ratio/low_mean": 8.020749664865434e-05, "clip_ratio/low_min": 8.020749664865434e-05, "clip_ratio/region_mean": 0.0005279623495880514, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1988.0, "completions/mean_length": 1478.239990234375, "completions/mean_terminated_length": 1317.5384521484375, "completions/min_length": 954.0, "completions/min_terminated_length": 954.0, "entropy": 0.17333095967769624, "epoch": 0.04076086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.016227753832936287, "learning_rate": 5e-05, "loss": 0.1112445518374443, "num_tokens": 1315440.0, "reward": -0.16179686784744263, "reward_std": 0.6155804395675659, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.7217968702316284, "rewards/length_penalty/std": 0.18375606834888458, "sampling/importance_sampling_ratio/max": 1.438353419303894, "sampling/importance_sampling_ratio/mean": 0.9937237501144409, "sampling/importance_sampling_ratio/min": 0.5849210619926453, "sampling/sampling_logp_difference/max": 0.5362783670425415, "sampling/sampling_logp_difference/mean": 0.0115585345774889, "step": 15, "step_time": 26.708784840768203 }, { "clip_ratio/high_max": 0.0006148368061985821, "clip_ratio/high_mean": 0.0006148368061985821, "clip_ratio/low_mean": 5.0958279462065545e-05, "clip_ratio/low_min": 5.0958279462065545e-05, "clip_ratio/region_mean": 0.000665795075474307, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 2000.0, "completions/mean_length": 1557.93994140625, "completions/mean_terminated_length": 1327.323486328125, "completions/min_length": 644.0, "completions/min_terminated_length": 644.0, "entropy": 0.25742628872394563, "epoch": 0.043478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.019402919337153435, "learning_rate": 5e-05, "loss": 0.10912851244211197, "num_tokens": 1397437.0, "reward": -0.08071289211511612, "reward_std": 0.6591700911521912, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.4712120592594147, "rewards/length_penalty/mean": -0.7607128620147705, "rewards/length_penalty/std": 0.23705528676509857, "sampling/importance_sampling_ratio/max": 1.7131775617599487, "sampling/importance_sampling_ratio/mean": 0.9908895492553711, "sampling/importance_sampling_ratio/min": 0.5364121198654175, "sampling/sampling_logp_difference/max": 0.6228525638580322, "sampling/sampling_logp_difference/mean": 0.016794202849268913, "step": 16, "step_time": 27.580034803133458 }, { "clip_ratio/high_max": 0.0004115843548788689, "clip_ratio/high_mean": 0.0004115843548788689, "clip_ratio/low_mean": 0.00015831452910788357, "clip_ratio/low_min": 0.00015831452910788357, "clip_ratio/region_mean": 0.000569898885441944, "completions/clipped_ratio": 0.5399999618530273, "completions/max_length": 2048.0, "completions/max_terminated_length": 1997.0, "completions/mean_length": 1762.719970703125, "completions/mean_terminated_length": 1427.826171875, "completions/min_length": 759.0, "completions/min_terminated_length": 759.0, "entropy": 0.2362867772579193, "epoch": 0.04619565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.01806093379855156, "learning_rate": 5e-05, "loss": 0.08567385375499725, "num_tokens": 1488223.0, "reward": -0.36070311069488525, "reward_std": 0.6595640778541565, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.8607031106948853, "rewards/length_penalty/std": 0.19437846541404724, "sampling/importance_sampling_ratio/max": 1.467071294784546, "sampling/importance_sampling_ratio/mean": 0.9918231964111328, "sampling/importance_sampling_ratio/min": 0.6348268985748291, "sampling/sampling_logp_difference/max": 0.4544029235839844, "sampling/sampling_logp_difference/mean": 0.015713775530457497, "step": 17, "step_time": 27.87512671900913 }, { "clip_ratio/high_max": 0.00046114472206681967, "clip_ratio/high_mean": 0.00046114472206681967, "clip_ratio/low_mean": 0.00015185942174866796, "clip_ratio/low_min": 0.00015185942174866796, "clip_ratio/region_mean": 0.0006130041554570198, "completions/clipped_ratio": 0.3799999952316284, "completions/max_length": 2048.0, "completions/max_terminated_length": 2048.0, "completions/mean_length": 1640.93994140625, "completions/mean_terminated_length": 1391.4515380859375, "completions/min_length": 894.0, "completions/min_terminated_length": 894.0, "entropy": 0.2142571061849594, "epoch": 0.04891304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01741565763950348, "learning_rate": 5e-05, "loss": 0.09045957028865814, "num_tokens": 1572770.0, "reward": -0.22124022245407104, "reward_std": 0.6729792952537537, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.8012402057647705, "rewards/length_penalty/std": 0.1984567791223526, "sampling/importance_sampling_ratio/max": 1.4769505262374878, "sampling/importance_sampling_ratio/mean": 0.992698073387146, "sampling/importance_sampling_ratio/min": 0.6199739575386047, "sampling/sampling_logp_difference/max": 0.47807788848876953, "sampling/sampling_logp_difference/mean": 0.014010032638907433, "step": 18, "step_time": 27.83921122015454 }, { "clip_ratio/high_max": 0.0015159809263423085, "clip_ratio/high_mean": 0.0015159809263423085, "clip_ratio/low_mean": 9.485194677836262e-05, "clip_ratio/low_min": 9.485194677836262e-05, "clip_ratio/region_mean": 0.0016108328709378839, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 1979.0, "completions/mean_length": 1699.02001953125, "completions/mean_terminated_length": 1401.74072265625, "completions/min_length": 602.0, "completions/min_terminated_length": 602.0, "entropy": 0.2789287269115448, "epoch": 0.051630434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.016779927536845207, "learning_rate": 5e-05, "loss": 0.1086677610874176, "num_tokens": 1661151.0, "reward": -0.28959959745407104, "reward_std": 0.6768404841423035, "rewards/correctness/mean": 0.5400000214576721, "rewards/correctness/std": 0.5034574270248413, "rewards/length_penalty/mean": -0.8295996189117432, "rewards/length_penalty/std": 0.2113669067621231, "sampling/importance_sampling_ratio/max": 1.910184621810913, "sampling/importance_sampling_ratio/mean": 0.990123987197876, "sampling/importance_sampling_ratio/min": 0.6096107959747314, "sampling/sampling_logp_difference/max": 0.6471998691558838, "sampling/sampling_logp_difference/mean": 0.017952606081962585, "step": 19, "step_time": 27.72973312623799 }, { "clip_ratio/high_max": 0.00045347435079747813, "clip_ratio/high_mean": 0.00045347435079747813, "clip_ratio/low_mean": 3.3310058643110094e-05, "clip_ratio/low_min": 3.3310058643110094e-05, "clip_ratio/region_mean": 0.0004867844079853967, "completions/clipped_ratio": 0.4599999785423279, "completions/max_length": 2048.0, "completions/max_terminated_length": 2016.0, "completions/mean_length": 1753.0, "completions/mean_terminated_length": 1501.7037353515625, "completions/min_length": 1014.0, "completions/min_terminated_length": 1014.0, "entropy": 0.2634665846824646, "epoch": 0.05434782608695652, "frac_reward_zero_std": 0.0, "grad_norm": 0.017920350655913353, "learning_rate": 5e-05, "loss": 0.11657360196113586, "num_tokens": 1751391.0, "reward": -0.2959570288658142, "reward_std": 0.6399221420288086, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.85595703125, "rewards/length_penalty/std": 0.16949063539505005, "sampling/importance_sampling_ratio/max": 1.601932168006897, "sampling/importance_sampling_ratio/mean": 0.9907858371734619, "sampling/importance_sampling_ratio/min": 0.5970116853713989, "sampling/sampling_logp_difference/max": 0.5158185958862305, "sampling/sampling_logp_difference/mean": 0.016703186556696892, "step": 20, "step_time": 28.113478776765987 }, { "clip_ratio/high_max": 0.0010554088861681522, "clip_ratio/high_mean": 0.0010554088861681522, "clip_ratio/low_mean": 7.666278688702732e-05, "clip_ratio/low_min": 7.666278688702732e-05, "clip_ratio/region_mean": 0.001132071705069393, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 1729.0, "completions/mean_length": 1603.02001953125, "completions/mean_terminated_length": 1393.61767578125, "completions/min_length": 917.0, "completions/min_terminated_length": 917.0, "entropy": 0.21676376163959504, "epoch": 0.057065217391304345, "frac_reward_zero_std": 0.0, "grad_norm": 0.01723313331604004, "learning_rate": 5e-05, "loss": 0.11572270095348358, "num_tokens": 1834952.0, "reward": -0.1027246043086052, "reward_std": 0.6278052926063538, "rewards/correctness/mean": 0.6800000071525574, "rewards/correctness/std": 0.47121208906173706, "rewards/length_penalty/mean": -0.7827246189117432, "rewards/length_penalty/std": 0.17379452288150787, "sampling/importance_sampling_ratio/max": 1.4738433361053467, "sampling/importance_sampling_ratio/mean": 0.9922135472297668, "sampling/importance_sampling_ratio/min": 0.5554258823394775, "sampling/sampling_logp_difference/max": 0.5880200862884521, "sampling/sampling_logp_difference/mean": 0.014427252113819122, "step": 21, "step_time": 27.254593407269567 }, { "clip_ratio/high_max": 0.0007700865971855819, "clip_ratio/high_mean": 0.0007700865971855819, "clip_ratio/low_mean": 0.00015520547749474644, "clip_ratio/low_min": 0.00015520547749474644, "clip_ratio/region_mean": 0.0009252920863218606, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1432.0999755859375, "completions/mean_terminated_length": 1258.3846435546875, "completions/min_length": 702.0, "completions/min_terminated_length": 702.0, "entropy": 0.25245159566402436, "epoch": 0.059782608695652176, "frac_reward_zero_std": 0.0, "grad_norm": 0.018060248345136642, "learning_rate": 5e-05, "loss": 0.12634620070457458, "num_tokens": 1911627.0, "reward": 0.08073242008686066, "reward_std": 0.604335367679596, "rewards/correctness/mean": 0.7799999713897705, "rewards/correctness/std": 0.4184519648551941, "rewards/length_penalty/mean": -0.699267566204071, "rewards/length_penalty/std": 0.23472638428211212, "sampling/importance_sampling_ratio/max": 1.5025248527526855, "sampling/importance_sampling_ratio/mean": 0.9911710619926453, "sampling/importance_sampling_ratio/min": 0.5105652213096619, "sampling/sampling_logp_difference/max": 0.6722369194030762, "sampling/sampling_logp_difference/mean": 0.016273191198706627, "step": 22, "step_time": 27.877674855990335 }, { "clip_ratio/high_max": 0.0010148974135518074, "clip_ratio/high_mean": 0.0010148974135518074, "clip_ratio/low_mean": 0.00019861414039041848, "clip_ratio/low_min": 0.00019861414039041848, "clip_ratio/region_mean": 0.0012135115335695445, "completions/clipped_ratio": 0.4399999976158142, "completions/max_length": 2048.0, "completions/max_terminated_length": 2043.0, "completions/mean_length": 1693.7799072265625, "completions/mean_terminated_length": 1415.46435546875, "completions/min_length": 782.0, "completions/min_terminated_length": 782.0, "entropy": 0.2882953703403473, "epoch": 0.0625, "frac_reward_zero_std": 0.0, "grad_norm": 0.018598053604364395, "learning_rate": 5e-05, "loss": 0.1307593435049057, "num_tokens": 2002436.0, "reward": -0.3670410215854645, "reward_std": 0.6679481863975525, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.8270410299301147, "rewards/length_penalty/std": 0.2002589851617813, "sampling/importance_sampling_ratio/max": 1.6674507856369019, "sampling/importance_sampling_ratio/mean": 0.9899606704711914, "sampling/importance_sampling_ratio/min": 0.4440256357192993, "sampling/sampling_logp_difference/max": 0.8118729591369629, "sampling/sampling_logp_difference/mean": 0.018604537472128868, "step": 23, "step_time": 28.789824944920838 }, { "clip_ratio/high_max": 0.0002756411849986762, "clip_ratio/high_mean": 0.0002756411849986762, "clip_ratio/low_mean": 0.00012162490529590286, "clip_ratio/low_min": 0.00012162490529590286, "clip_ratio/region_mean": 0.00039726609247736633, "completions/clipped_ratio": 0.47999998927116394, "completions/max_length": 2048.0, "completions/max_terminated_length": 1958.0, "completions/mean_length": 1646.0, "completions/mean_terminated_length": 1274.923095703125, "completions/min_length": 710.0, "completions/min_terminated_length": 710.0, "entropy": 0.2541309058666229, "epoch": 0.06521739130434782, "frac_reward_zero_std": 0.0, "grad_norm": 0.017318135127425194, "learning_rate": 5e-05, "loss": 0.09953144192695618, "num_tokens": 2087026.0, "reward": -0.3637109398841858, "reward_std": 0.7076661586761475, "rewards/correctness/mean": 0.4399999976158142, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.8037109375, "rewards/length_penalty/std": 0.23487527668476105, "sampling/importance_sampling_ratio/max": 1.5337666273117065, "sampling/importance_sampling_ratio/mean": 0.9909667372703552, "sampling/importance_sampling_ratio/min": 0.3068637549877167, "sampling/sampling_logp_difference/max": 1.1813514232635498, "sampling/sampling_logp_difference/mean": 0.01644875854253769, "step": 24, "step_time": 27.854392275679857 }, { "clip_ratio/high_max": 0.00038455752946902065, "clip_ratio/high_mean": 0.00038455752946902065, "clip_ratio/low_mean": 0.00010470138513483108, "clip_ratio/low_min": 0.00010470138513483108, "clip_ratio/region_mean": 0.000489258929155767, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1733.0, "completions/mean_length": 1324.659912109375, "completions/mean_terminated_length": 1143.8250732421875, "completions/min_length": 619.0, "completions/min_terminated_length": 619.0, "entropy": 0.2677806198596954, "epoch": 0.06793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.017140235751867294, "learning_rate": 5e-05, "loss": 0.10585936903953552, "num_tokens": 2155659.0, "reward": 0.15319335460662842, "reward_std": 0.5960436463356018, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.6468066573143005, "rewards/length_penalty/std": 0.21872861683368683, "sampling/importance_sampling_ratio/max": 1.779210090637207, "sampling/importance_sampling_ratio/mean": 0.9906288385391235, "sampling/importance_sampling_ratio/min": 0.421924889087677, "sampling/sampling_logp_difference/max": 0.862928032875061, "sampling/sampling_logp_difference/mean": 0.01819814182817936, "step": 25, "step_time": 26.028295453172177 }, { "clip_ratio/high_max": 0.0008262786956038326, "clip_ratio/high_mean": 0.0008262786956038326, "clip_ratio/low_mean": 0.0001230211986694485, "clip_ratio/low_min": 0.0001230211986694485, "clip_ratio/region_mean": 0.0009492998942732811, "completions/clipped_ratio": 0.0, "completions/max_length": 1891.0, "completions/max_terminated_length": 1891.0, "completions/mean_length": 1210.5399169921875, "completions/mean_terminated_length": 1210.5399169921875, "completions/min_length": 735.0, "completions/min_terminated_length": 735.0, "entropy": 0.21406630873680116, "epoch": 0.07065217391304347, "frac_reward_zero_std": 0.0, "grad_norm": 0.01839393749833107, "learning_rate": 5e-05, "loss": 0.11455884575843811, "num_tokens": 2218656.0, "reward": 0.008916015736758709, "reward_std": 0.46137112379074097, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5910840034484863, "rewards/length_penalty/std": 0.13499784469604492, "sampling/importance_sampling_ratio/max": 1.9276258945465088, "sampling/importance_sampling_ratio/mean": 0.992218554019928, "sampling/importance_sampling_ratio/min": 0.3955056071281433, "sampling/sampling_logp_difference/max": 0.9275903701782227, "sampling/sampling_logp_difference/mean": 0.015513994731009007, "step": 26, "step_time": 23.871269115945324 }, { "clip_ratio/high_max": 0.00048658672021701933, "clip_ratio/high_mean": 0.00048658672021701933, "clip_ratio/low_mean": 0.00018206397944595665, "clip_ratio/low_min": 0.00018206397944595665, "clip_ratio/region_mean": 0.0006686506967525929, "completions/clipped_ratio": 0.3199999928474426, "completions/max_length": 2048.0, "completions/max_terminated_length": 2014.0, "completions/mean_length": 1442.5, "completions/mean_terminated_length": 1157.558837890625, "completions/min_length": 249.0, "completions/min_terminated_length": 249.0, "entropy": 0.20492501854896544, "epoch": 0.07336956521739131, "frac_reward_zero_std": 0.0, "grad_norm": 0.016834646463394165, "learning_rate": 5e-05, "loss": 0.08778552711009979, "num_tokens": 2293031.0, "reward": -0.004345702938735485, "reward_std": 0.6843211054801941, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.704345703125, "rewards/length_penalty/std": 0.27017921209335327, "sampling/importance_sampling_ratio/max": 2.6669118404388428, "sampling/importance_sampling_ratio/mean": 0.9927846789360046, "sampling/importance_sampling_ratio/min": 0.2942550778388977, "sampling/sampling_logp_difference/max": 1.2233083248138428, "sampling/sampling_logp_difference/mean": 0.01452636905014515, "step": 27, "step_time": 26.963234761962667 }, { "clip_ratio/high_max": 0.0006381792773026973, "clip_ratio/high_mean": 0.0006381792773026973, "clip_ratio/low_mean": 5.17316599143669e-05, "clip_ratio/low_min": 5.17316599143669e-05, "clip_ratio/region_mean": 0.0006899109343066811, "completions/clipped_ratio": 0.0, "completions/max_length": 2010.0, "completions/max_terminated_length": 2010.0, "completions/mean_length": 1171.260009765625, "completions/mean_terminated_length": 1171.260009765625, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.17487574517726898, "epoch": 0.07608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01512145809829235, "learning_rate": 5e-05, "loss": 0.08589713275432587, "num_tokens": 2354954.0, "reward": 0.22809569537639618, "reward_std": 0.41687142848968506, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.5719043016433716, "rewards/length_penalty/std": 0.16128839552402496, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9937527775764465, "sampling/importance_sampling_ratio/min": 0.34299761056900024, "sampling/sampling_logp_difference/max": 1.6456990242004395, "sampling/sampling_logp_difference/mean": 0.013332147151231766, "step": 28, "step_time": 24.870024599833414 }, { "clip_ratio/high_max": 0.0007338077353779227, "clip_ratio/high_mean": 0.0007338077353779227, "clip_ratio/low_mean": 8.471053806715644e-05, "clip_ratio/low_min": 8.471053806715644e-05, "clip_ratio/region_mean": 0.0008185182814486325, "completions/clipped_ratio": 0.3999999761581421, "completions/max_length": 2048.0, "completions/max_terminated_length": 2026.0, "completions/mean_length": 1596.6199951171875, "completions/mean_terminated_length": 1295.7000732421875, "completions/min_length": 576.0, "completions/min_terminated_length": 576.0, "entropy": 0.23987057507038118, "epoch": 0.07880434782608696, "frac_reward_zero_std": 0.0, "grad_norm": 0.019053541123867035, "learning_rate": 5e-05, "loss": 0.08386198431253433, "num_tokens": 2438225.0, "reward": -0.1795995980501175, "reward_std": 0.7008415460586548, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.7795996069908142, "rewards/length_penalty/std": 0.2576085925102234, "sampling/importance_sampling_ratio/max": 2.1604795455932617, "sampling/importance_sampling_ratio/mean": 0.9914707541465759, "sampling/importance_sampling_ratio/min": 0.36934080719947815, "sampling/sampling_logp_difference/max": 0.9960354566574097, "sampling/sampling_logp_difference/mean": 0.016715386882424355, "step": 29, "step_time": 28.12866471405141 }, { "clip_ratio/high_max": 0.0006765315891243517, "clip_ratio/high_mean": 0.0006765315891243517, "clip_ratio/low_mean": 6.585299561265857e-05, "clip_ratio/low_min": 6.585299561265857e-05, "clip_ratio/region_mean": 0.0007423845818266273, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 2033.0, "completions/mean_length": 1187.1600341796875, "completions/mean_terminated_length": 1132.2127685546875, "completions/min_length": 709.0, "completions/min_terminated_length": 709.0, "entropy": 0.16735150218009948, "epoch": 0.08152173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.016750315204262733, "learning_rate": 5e-05, "loss": 0.13103985786437988, "num_tokens": 2499373.0, "reward": 0.38033202290534973, "reward_std": 0.32468104362487793, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.5796679854393005, "rewards/length_penalty/std": 0.17866531014442444, "sampling/importance_sampling_ratio/max": 2.3867034912109375, "sampling/importance_sampling_ratio/mean": 0.9940452575683594, "sampling/importance_sampling_ratio/min": 0.294817179441452, "sampling/sampling_logp_difference/max": 1.2213997840881348, "sampling/sampling_logp_difference/mean": 0.013628384098410606, "step": 30, "step_time": 25.21493199793622 }, { "clip_ratio/high_max": 0.000489990774076432, "clip_ratio/high_mean": 0.000489990774076432, "clip_ratio/low_mean": 7.639809919055552e-05, "clip_ratio/low_min": 7.639809919055552e-05, "clip_ratio/region_mean": 0.0005663888703566044, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 1981.0, "completions/mean_length": 1442.97998046875, "completions/mean_terminated_length": 1360.477294921875, "completions/min_length": 540.0, "completions/min_terminated_length": 540.0, "entropy": 0.17727571427822114, "epoch": 0.08423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.01648411899805069, "learning_rate": 5e-05, "loss": 0.09550511837005615, "num_tokens": 2574652.0, "reward": -0.2245800793170929, "reward_std": 0.48963484168052673, "rewards/correctness/mean": 0.47999998927116394, "rewards/correctness/std": 0.5046720504760742, "rewards/length_penalty/mean": -0.7045800685882568, "rewards/length_penalty/std": 0.20444612205028534, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933596849441528, "sampling/importance_sampling_ratio/min": 0.28033190965652466, "sampling/sampling_logp_difference/max": 1.6622023582458496, "sampling/sampling_logp_difference/mean": 0.013714849948883057, "step": 31, "step_time": 27.176624103216454 }, { "clip_ratio/high_max": 0.000510922638932243, "clip_ratio/high_mean": 0.000510922638932243, "clip_ratio/low_mean": 0.00010051408171420916, "clip_ratio/low_min": 0.00010051408171420916, "clip_ratio/region_mean": 0.0006114367221016437, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1575.0, "completions/mean_length": 999.5199584960938, "completions/mean_terminated_length": 978.1224365234375, "completions/min_length": 503.0, "completions/min_terminated_length": 503.0, "entropy": 0.20394009947776795, "epoch": 0.08695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.016988106071949005, "learning_rate": 5e-05, "loss": 0.07811405509710312, "num_tokens": 2626558.0, "reward": 0.431953102350235, "reward_std": 0.3679496645927429, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404749393463135, "rewards/length_penalty/mean": -0.48804688453674316, "rewards/length_penalty/std": 0.15092554688453674, "sampling/importance_sampling_ratio/max": 2.677778959274292, "sampling/importance_sampling_ratio/mean": 0.9927431344985962, "sampling/importance_sampling_ratio/min": 0.1728629618883133, "sampling/sampling_logp_difference/max": 1.755256175994873, "sampling/sampling_logp_difference/mean": 0.016466621309518814, "step": 32, "step_time": 24.21077091898769 }, { "clip_ratio/high_max": 0.0005299294571159407, "clip_ratio/high_mean": 0.0005299294571159407, "clip_ratio/low_mean": 0.00016564234683755786, "clip_ratio/low_min": 0.00016564234683755786, "clip_ratio/region_mean": 0.0006955718214157969, "completions/clipped_ratio": 0.25999999046325684, "completions/max_length": 2048.0, "completions/max_terminated_length": 1875.0, "completions/mean_length": 1336.719970703125, "completions/mean_terminated_length": 1086.810791015625, "completions/min_length": 648.0, "completions/min_terminated_length": 648.0, "entropy": 0.19695305824279785, "epoch": 0.08967391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.01601804420351982, "learning_rate": 5e-05, "loss": 0.05671275779604912, "num_tokens": 2697034.0, "reward": -0.09269531071186066, "reward_std": 0.6529631018638611, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.6526952981948853, "rewards/length_penalty/std": 0.26207444071769714, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930840134620667, "sampling/importance_sampling_ratio/min": 0.2735775411128998, "sampling/sampling_logp_difference/max": 1.3018450736999512, "sampling/sampling_logp_difference/mean": 0.01516858022660017, "step": 33, "step_time": 26.30819136183709 }, { "clip_ratio/high_max": 0.0012457524426281452, "clip_ratio/high_mean": 0.0012457524426281452, "clip_ratio/low_mean": 0.00011662203760351986, "clip_ratio/low_min": 0.00011662203760351986, "clip_ratio/region_mean": 0.0013623744598589838, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 1402.179931640625, "completions/mean_terminated_length": 1240.7249755859375, "completions/min_length": 251.0, "completions/min_terminated_length": 251.0, "entropy": 0.24012117385864257, "epoch": 0.09239130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.018035899847745895, "learning_rate": 5e-05, "loss": 0.14534080028533936, "num_tokens": 2771033.0, "reward": 0.11534179747104645, "reward_std": 0.5863871574401855, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.6846582293510437, "rewards/length_penalty/std": 0.22775794565677643, "sampling/importance_sampling_ratio/max": 2.5232789516448975, "sampling/importance_sampling_ratio/mean": 0.9910904169082642, "sampling/importance_sampling_ratio/min": 0.13666574656963348, "sampling/sampling_logp_difference/max": 1.9902172088623047, "sampling/sampling_logp_difference/mean": 0.018044251948595047, "step": 34, "step_time": 27.28829189715907 }, { "clip_ratio/high_max": 0.0008087092312052846, "clip_ratio/high_mean": 0.0008087092312052846, "clip_ratio/low_mean": 0.0001302991237025708, "clip_ratio/low_min": 0.0001302991237025708, "clip_ratio/region_mean": 0.0009390083723701537, "completions/clipped_ratio": 0.1599999964237213, "completions/max_length": 2048.0, "completions/max_terminated_length": 1989.0, "completions/mean_length": 1313.5999755859375, "completions/mean_terminated_length": 1173.71435546875, "completions/min_length": 699.0, "completions/min_terminated_length": 699.0, "entropy": 0.2610628128051758, "epoch": 0.09510869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.022161388769745827, "learning_rate": 5e-05, "loss": 0.15209972858428955, "num_tokens": 2839443.0, "reward": 0.1785937398672104, "reward_std": 0.5776634812355042, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.641406238079071, "rewards/length_penalty/std": 0.23114332556724548, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9906759262084961, "sampling/importance_sampling_ratio/min": 0.13394932448863983, "sampling/sampling_logp_difference/max": 2.01029372215271, "sampling/sampling_logp_difference/mean": 0.020522888749837875, "step": 35, "step_time": 26.005766293033957 }, { "clip_ratio/high_max": 0.0005731199169531464, "clip_ratio/high_mean": 0.0005731199169531464, "clip_ratio/low_mean": 3.850940120173618e-05, "clip_ratio/low_min": 3.850940120173618e-05, "clip_ratio/region_mean": 0.000611629307968542, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 1934.0, "completions/mean_length": 1033.0, "completions/mean_terminated_length": 867.7674560546875, "completions/min_length": 533.0, "completions/min_terminated_length": 533.0, "entropy": 0.19119977653026582, "epoch": 0.09782608695652174, "frac_reward_zero_std": 0.0, "grad_norm": 0.015223798342049122, "learning_rate": 5e-05, "loss": 0.06148340925574303, "num_tokens": 2893563.0, "reward": 0.3156054615974426, "reward_std": 0.6145754456520081, "rewards/correctness/mean": 0.8199999928474426, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.50439453125, "rewards/length_penalty/std": 0.24171145260334015, "sampling/importance_sampling_ratio/max": 2.829972267150879, "sampling/importance_sampling_ratio/mean": 0.9929235577583313, "sampling/importance_sampling_ratio/min": 0.04968615993857384, "sampling/sampling_logp_difference/max": 3.0020289421081543, "sampling/sampling_logp_difference/mean": 0.01784489117562771, "step": 36, "step_time": 24.962291612988338 }, { "clip_ratio/high_max": 0.0014889343758113683, "clip_ratio/high_mean": 0.0014889343758113683, "clip_ratio/low_mean": 3.582693316275254e-05, "clip_ratio/low_min": 3.582693316275254e-05, "clip_ratio/region_mean": 0.0015247612958773971, "completions/clipped_ratio": 0.2199999988079071, "completions/max_length": 2048.0, "completions/max_terminated_length": 1970.0, "completions/mean_length": 1215.8599853515625, "completions/mean_terminated_length": 981.1538696289062, "completions/min_length": 666.0, "completions/min_terminated_length": 666.0, "entropy": 0.24190285801887512, "epoch": 0.10054347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.017842410132288933, "learning_rate": 5e-05, "loss": 0.08806680142879486, "num_tokens": 2957876.0, "reward": -0.013681639917194843, "reward_std": 0.6404426097869873, "rewards/correctness/mean": 0.5799999833106995, "rewards/correctness/std": 0.49856939911842346, "rewards/length_penalty/mean": -0.5936816334724426, "rewards/length_penalty/std": 0.24957187473773956, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9916677474975586, "sampling/importance_sampling_ratio/min": 0.04188256710767746, "sampling/sampling_logp_difference/max": 3.1728856563568115, "sampling/sampling_logp_difference/mean": 0.019735973328351974, "step": 37, "step_time": 26.66670376714319 }, { "clip_ratio/high_max": 0.0004470044106710702, "clip_ratio/high_mean": 0.0004470044106710702, "clip_ratio/low_mean": 0.00011004692350979894, "clip_ratio/low_min": 0.00011004692350979894, "clip_ratio/region_mean": 0.0005570513429120183, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 2030.0, "completions/mean_length": 1304.7999267578125, "completions/mean_terminated_length": 1183.81396484375, "completions/min_length": 591.0, "completions/min_terminated_length": 591.0, "entropy": 0.17063663005828858, "epoch": 0.10326086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.018432138487696648, "learning_rate": 5e-05, "loss": 0.10655094683170319, "num_tokens": 3026506.0, "reward": 0.20289061963558197, "reward_std": 0.5410789251327515, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.6371093988418579, "rewards/length_penalty/std": 0.25031739473342896, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9935672879219055, "sampling/importance_sampling_ratio/min": 0.04130437225103378, "sampling/sampling_logp_difference/max": 3.1867868900299072, "sampling/sampling_logp_difference/mean": 0.015192362479865551, "step": 38, "step_time": 26.10579670360312 }, { "clip_ratio/high_max": 0.0003236483491491526, "clip_ratio/high_mean": 0.0003236483491491526, "clip_ratio/low_mean": 7.918733317637816e-05, "clip_ratio/low_min": 7.918733317637816e-05, "clip_ratio/region_mean": 0.00040283568087033926, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1583.0, "completions/mean_length": 1134.47998046875, "completions/mean_terminated_length": 906.1000366210938, "completions/min_length": 504.0, "completions/min_terminated_length": 504.0, "entropy": 0.17430886328220369, "epoch": 0.10597826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.01502721756696701, "learning_rate": 5e-05, "loss": 0.04892420396208763, "num_tokens": 3085720.0, "reward": 0.04605468735098839, "reward_std": 0.7434121966362, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.5539453029632568, "rewards/length_penalty/std": 0.2671849727630615, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9934588074684143, "sampling/importance_sampling_ratio/min": 0.1332322210073471, "sampling/sampling_logp_difference/max": 2.0156617164611816, "sampling/sampling_logp_difference/mean": 0.014964032918214798, "step": 39, "step_time": 25.17327128793113 }, { "clip_ratio/high_max": 0.00035759085731115194, "clip_ratio/high_mean": 0.00035759085731115194, "clip_ratio/low_mean": 0.00014421666273847222, "clip_ratio/low_min": 0.00014421666273847222, "clip_ratio/region_mean": 0.0005018075375119224, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 869.4400024414062, "completions/mean_terminated_length": 845.3877563476562, "completions/min_length": 550.0, "completions/min_terminated_length": 550.0, "entropy": 0.18513228595256806, "epoch": 0.10869565217391304, "frac_reward_zero_std": 0.0, "grad_norm": 0.017670467495918274, "learning_rate": 5e-05, "loss": 0.07665302604436874, "num_tokens": 3131912.0, "reward": 0.5354687571525574, "reward_std": 0.31231585144996643, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.4245312511920929, "rewards/length_penalty/std": 0.13832274079322815, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9931167364120483, "sampling/importance_sampling_ratio/min": 0.0789855495095253, "sampling/sampling_logp_difference/max": 2.5384902954101562, "sampling/sampling_logp_difference/mean": 0.01847790740430355, "step": 40, "step_time": 24.25951893720776 }, { "clip_ratio/high_max": 0.0005802025028970092, "clip_ratio/high_mean": 0.0005802025028970092, "clip_ratio/low_mean": 9.926969069056213e-05, "clip_ratio/low_min": 9.926969069056213e-05, "clip_ratio/region_mean": 0.0006794722052291035, "completions/clipped_ratio": 0.07999999821186066, "completions/max_length": 2048.0, "completions/max_terminated_length": 2042.0, "completions/mean_length": 1125.52001953125, "completions/mean_terminated_length": 1045.3043212890625, "completions/min_length": 588.0, "completions/min_terminated_length": 588.0, "entropy": 0.1967887282371521, "epoch": 0.11141304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.019327964633703232, "learning_rate": 5e-05, "loss": 0.09903993457555771, "num_tokens": 3191928.0, "reward": 0.01042968686670065, "reward_std": 0.5763521790504456, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.5495703220367432, "rewards/length_penalty/std": 0.1971028596162796, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9928300976753235, "sampling/importance_sampling_ratio/min": 0.141234889626503, "sampling/sampling_logp_difference/max": 1.9573308229446411, "sampling/sampling_logp_difference/mean": 0.017453640699386597, "step": 41, "step_time": 25.61839163210243 }, { "clip_ratio/high_max": 0.00043595051392912864, "clip_ratio/high_mean": 0.00043595051392912864, "clip_ratio/low_mean": 0.00010612716432660818, "clip_ratio/low_min": 0.00010612716432660818, "clip_ratio/region_mean": 0.0005420776724349707, "completions/clipped_ratio": 0.14000000059604645, "completions/max_length": 2048.0, "completions/max_terminated_length": 2030.0, "completions/mean_length": 1083.97998046875, "completions/mean_terminated_length": 927.0465087890625, "completions/min_length": 386.0, "completions/min_terminated_length": 386.0, "entropy": 0.197939333319664, "epoch": 0.11413043478260869, "frac_reward_zero_std": 0.0, "grad_norm": 0.018111297860741615, "learning_rate": 5e-05, "loss": 0.10085495561361313, "num_tokens": 3249117.0, "reward": 0.3307128846645355, "reward_std": 0.5710539221763611, "rewards/correctness/mean": 0.8600000143051147, "rewards/correctness/std": 0.3505098223686218, "rewards/length_penalty/mean": -0.5292870998382568, "rewards/length_penalty/std": 0.26221367716789246, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925159215927124, "sampling/importance_sampling_ratio/min": 0.059626415371894836, "sampling/sampling_logp_difference/max": 2.8196566104888916, "sampling/sampling_logp_difference/mean": 0.017921241000294685, "step": 42, "step_time": 25.15766688482836 }, { "clip_ratio/high_max": 0.0006175725313369185, "clip_ratio/high_mean": 0.0006175725313369185, "clip_ratio/low_mean": 0.00010935824830085039, "clip_ratio/low_min": 0.00010935824830085039, "clip_ratio/region_mean": 0.000726930785458535, "completions/clipped_ratio": 0.0, "completions/max_length": 1572.0, "completions/max_terminated_length": 1572.0, "completions/mean_length": 744.5399780273438, "completions/mean_terminated_length": 744.5399780273438, "completions/min_length": 410.0, "completions/min_terminated_length": 410.0, "entropy": 0.20578812658786774, "epoch": 0.11684782608695653, "frac_reward_zero_std": 0.0, "grad_norm": 0.016133708879351616, "learning_rate": 5e-05, "loss": 0.06198643147945404, "num_tokens": 3288314.0, "reward": 0.5764550566673279, "reward_std": 0.2711663842201233, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.36354491114616394, "rewards/length_penalty/std": 0.12007534503936768, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9924009442329407, "sampling/importance_sampling_ratio/min": 0.05938252806663513, "sampling/sampling_logp_difference/max": 2.8237552642822266, "sampling/sampling_logp_difference/mean": 0.021952394396066666, "step": 43, "step_time": 18.829215484438464 }, { "clip_ratio/high_max": 0.0006127148182713427, "clip_ratio/high_mean": 0.0006127148182713427, "clip_ratio/low_mean": 7.67521167290397e-05, "clip_ratio/low_min": 7.67521167290397e-05, "clip_ratio/region_mean": 0.0006894669437315315, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1985.0, "completions/mean_length": 1012.1599731445312, "completions/mean_terminated_length": 969.0, "completions/min_length": 462.0, "completions/min_terminated_length": 462.0, "entropy": 0.18495380580425264, "epoch": 0.11956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.016027262434363365, "learning_rate": 5e-05, "loss": 0.11064858734607697, "num_tokens": 3341922.0, "reward": 0.46578124165534973, "reward_std": 0.35385408997535706, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.49421873688697815, "rewards/length_penalty/std": 0.21152016520500183, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9924459457397461, "sampling/importance_sampling_ratio/min": 0.0823984369635582, "sampling/sampling_logp_difference/max": 2.4961888790130615, "sampling/sampling_logp_difference/mean": 0.01826482079923153, "step": 44, "step_time": 24.763890769099817 }, { "clip_ratio/high_max": 0.0009404713404364884, "clip_ratio/high_mean": 0.0009404713404364884, "clip_ratio/low_mean": 0.00015583014173898847, "clip_ratio/low_min": 0.00015583014173898847, "clip_ratio/region_mean": 0.001096301490906626, "completions/clipped_ratio": 0.11999999731779099, "completions/max_length": 2048.0, "completions/max_terminated_length": 2028.0, "completions/mean_length": 1041.1199951171875, "completions/mean_terminated_length": 903.8182373046875, "completions/min_length": 346.0, "completions/min_terminated_length": 346.0, "entropy": 0.2129930168390274, "epoch": 0.12228260869565218, "frac_reward_zero_std": 0.0, "grad_norm": 0.01909661665558815, "learning_rate": 5e-05, "loss": 0.09125189483165741, "num_tokens": 3396918.0, "reward": 0.23164062201976776, "reward_std": 0.65437251329422, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.5083593726158142, "rewards/length_penalty/std": 0.2664102613925934, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9919946789741516, "sampling/importance_sampling_ratio/min": 0.11528640240430832, "sampling/sampling_logp_difference/max": 2.376345634460449, "sampling/sampling_logp_difference/mean": 0.019899416714906693, "step": 45, "step_time": 24.942179046105593 }, { "clip_ratio/high_max": 0.0012106924899853767, "clip_ratio/high_mean": 0.0012106924899853767, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0012106924899853767, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1415.0, "completions/mean_length": 1034.919921875, "completions/mean_terminated_length": 781.6500244140625, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.22283259630203248, "epoch": 0.125, "frac_reward_zero_std": 0.0, "grad_norm": 0.015132801607251167, "learning_rate": 5e-05, "loss": 0.05163145065307617, "num_tokens": 3452174.0, "reward": 0.29466795921325684, "reward_std": 0.660883367061615, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.5053320527076721, "rewards/length_penalty/std": 0.2675746977329254, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9920194745063782, "sampling/importance_sampling_ratio/min": 0.06478776037693024, "sampling/sampling_logp_difference/max": 2.7366385459899902, "sampling/sampling_logp_difference/mean": 0.020383715629577637, "step": 46, "step_time": 25.52300767414272 }, { "clip_ratio/high_max": 0.0007857588323531672, "clip_ratio/high_mean": 0.0007857588323531672, "clip_ratio/low_mean": 4.7264108434319493e-05, "clip_ratio/low_min": 4.7264108434319493e-05, "clip_ratio/region_mean": 0.0008330229320563376, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1562.0, "completions/mean_length": 782.1799926757812, "completions/mean_terminated_length": 729.4375, "completions/min_length": 482.0, "completions/min_terminated_length": 482.0, "entropy": 0.18785084784030914, "epoch": 0.12771739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.017658179625868797, "learning_rate": 5e-05, "loss": 0.06419933587312698, "num_tokens": 3493673.0, "reward": 0.5580761432647705, "reward_std": 0.3772341310977936, "rewards/correctness/mean": 0.9399999976158142, "rewards/correctness/std": 0.2398979365825653, "rewards/length_penalty/mean": -0.3819238245487213, "rewards/length_penalty/std": 0.1710575968027115, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933866262435913, "sampling/importance_sampling_ratio/min": 0.10128559172153473, "sampling/sampling_logp_difference/max": 2.289811134338379, "sampling/sampling_logp_difference/mean": 0.019509905949234962, "step": 47, "step_time": 23.746050203917548 }, { "clip_ratio/high_max": 0.0007406561751849949, "clip_ratio/high_mean": 0.0007406561751849949, "clip_ratio/low_mean": 0.00021931264636805282, "clip_ratio/low_min": 0.00021931264636805282, "clip_ratio/region_mean": 0.0009599688230082393, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1944.0, "completions/mean_length": 1003.0599975585938, "completions/mean_terminated_length": 959.5208740234375, "completions/min_length": 340.0, "completions/min_terminated_length": 340.0, "entropy": 0.16566215753555297, "epoch": 0.13043478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.01728041097521782, "learning_rate": 5e-05, "loss": 0.08843446522951126, "num_tokens": 3546786.0, "reward": 0.25022459030151367, "reward_std": 0.5651769638061523, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.4897753894329071, "rewards/length_penalty/std": 0.24563182890415192, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9940137267112732, "sampling/importance_sampling_ratio/min": 0.03814614936709404, "sampling/sampling_logp_difference/max": 3.2663304805755615, "sampling/sampling_logp_difference/mean": 0.017063232138752937, "step": 48, "step_time": 24.783017989015207 }, { "clip_ratio/high_max": 0.00027117388090118765, "clip_ratio/high_mean": 0.00027117388090118765, "clip_ratio/low_mean": 0.00021409733890322968, "clip_ratio/low_min": 0.00021409733890322968, "clip_ratio/region_mean": 0.0004852712154388428, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1468.0, "completions/mean_length": 976.1599731445312, "completions/mean_terminated_length": 708.2000122070312, "completions/min_length": 415.0, "completions/min_terminated_length": 415.0, "entropy": 0.23387743532657623, "epoch": 0.1331521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.01643521524965763, "learning_rate": 5e-05, "loss": 0.06901802122592926, "num_tokens": 3598284.0, "reward": 0.3233593702316284, "reward_std": 0.6772034764289856, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.40406104922294617, "rewards/length_penalty/mean": -0.47664061188697815, "rewards/length_penalty/std": 0.28587260842323303, "sampling/importance_sampling_ratio/max": 2.514124870300293, "sampling/importance_sampling_ratio/mean": 0.9907704591751099, "sampling/importance_sampling_ratio/min": 0.078238345682621, "sampling/sampling_logp_difference/max": 2.5479953289031982, "sampling/sampling_logp_difference/mean": 0.022797785699367523, "step": 49, "step_time": 24.643268001032993 }, { "clip_ratio/high_max": 0.0015754754422232509, "clip_ratio/high_mean": 0.0015754754422232509, "clip_ratio/low_mean": 4.226989403832704e-05, "clip_ratio/low_min": 4.226989403832704e-05, "clip_ratio/region_mean": 0.0016177453333511948, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 830.0, "completions/mean_length": 856.1199951171875, "completions/mean_terminated_length": 558.1500244140625, "completions/min_length": 257.0, "completions/min_terminated_length": 257.0, "entropy": 0.23729664981365203, "epoch": 0.1358695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.012057525105774403, "learning_rate": 5e-05, "loss": 0.0186992846429348, "num_tokens": 3643500.0, "reward": 0.1819726526737213, "reward_std": 0.7392576932907104, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.4180273413658142, "rewards/length_penalty/std": 0.30474045872688293, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9907179474830627, "sampling/importance_sampling_ratio/min": 0.04562351852655411, "sampling/sampling_logp_difference/max": 3.087332010269165, "sampling/sampling_logp_difference/mean": 0.022797871381044388, "step": 50, "step_time": 24.104078602744266 }, { "clip_ratio/high_max": 0.0010256923327688129, "clip_ratio/high_mean": 0.0010256923327688129, "clip_ratio/low_mean": 5.421792156994343e-05, "clip_ratio/low_min": 5.421792156994343e-05, "clip_ratio/region_mean": 0.0010799102543387561, "completions/clipped_ratio": 0.0, "completions/max_length": 1711.0, "completions/max_terminated_length": 1711.0, "completions/mean_length": 729.0599975585938, "completions/mean_terminated_length": 729.0599975585938, "completions/min_length": 316.0, "completions/min_terminated_length": 316.0, "entropy": 0.1816806674003601, "epoch": 0.13858695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01620478369295597, "learning_rate": 5e-05, "loss": 0.051869865506887436, "num_tokens": 3682463.0, "reward": 0.6440136432647705, "reward_std": 0.14435778558254242, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.3559863269329071, "rewards/length_penalty/std": 0.14435778558254242, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930160641670227, "sampling/importance_sampling_ratio/min": 0.06998127698898315, "sampling/sampling_logp_difference/max": 2.659527540206909, "sampling/sampling_logp_difference/mean": 0.02179049886763096, "step": 51, "step_time": 19.07612481410615 }, { "clip_ratio/high_max": 0.0005590053682681173, "clip_ratio/high_mean": 0.0005590053682681173, "clip_ratio/low_mean": 6.485775811597705e-05, "clip_ratio/low_min": 6.485775811597705e-05, "clip_ratio/region_mean": 0.0006238631205633283, "completions/clipped_ratio": 0.0, "completions/max_length": 1750.0, "completions/max_terminated_length": 1750.0, "completions/mean_length": 686.7000122070312, "completions/mean_terminated_length": 686.7000122070312, "completions/min_length": 375.0, "completions/min_terminated_length": 375.0, "entropy": 0.15555862784385682, "epoch": 0.14130434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.015548508614301682, "learning_rate": 5e-05, "loss": 0.046372976154088974, "num_tokens": 3719058.0, "reward": 0.4646972417831421, "reward_std": 0.4265141785144806, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.3353027403354645, "rewards/length_penalty/std": 0.1275048404932022, "sampling/importance_sampling_ratio/max": 2.523122787475586, "sampling/importance_sampling_ratio/mean": 0.994541585445404, "sampling/importance_sampling_ratio/min": 0.18221420049667358, "sampling/sampling_logp_difference/max": 1.7025723457336426, "sampling/sampling_logp_difference/mean": 0.02134961448609829, "step": 52, "step_time": 19.41129402187653 }, { "clip_ratio/high_max": 0.0006675232667475939, "clip_ratio/high_mean": 0.0006675232667475939, "clip_ratio/low_mean": 0.00011539483675733208, "clip_ratio/low_min": 0.00011539483675733208, "clip_ratio/region_mean": 0.0007829181035049259, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1851.0, "completions/mean_length": 791.739990234375, "completions/mean_terminated_length": 739.3958740234375, "completions/min_length": 304.0, "completions/min_terminated_length": 304.0, "entropy": 0.19051934480667115, "epoch": 0.14402173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.018420696258544922, "learning_rate": 5e-05, "loss": 0.13225115835666656, "num_tokens": 3761365.0, "reward": 0.3734081983566284, "reward_std": 0.5679783225059509, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.3865917921066284, "rewards/length_penalty/std": 0.21445991098880768, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9924401640892029, "sampling/importance_sampling_ratio/min": 0.06539883464574814, "sampling/sampling_logp_difference/max": 2.727250814437866, "sampling/sampling_logp_difference/mean": 0.02331310324370861, "step": 53, "step_time": 23.712469375226647 }, { "clip_ratio/high_max": 0.00040742788987699894, "clip_ratio/high_mean": 0.00040742788987699894, "clip_ratio/low_mean": 8.799331262707711e-05, "clip_ratio/low_min": 8.799331262707711e-05, "clip_ratio/region_mean": 0.000495421202504076, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1800.0, "completions/mean_length": 909.1199951171875, "completions/mean_terminated_length": 782.5778198242188, "completions/min_length": 404.0, "completions/min_terminated_length": 404.0, "entropy": 0.20038393437862395, "epoch": 0.14673913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.01630176417529583, "learning_rate": 5e-05, "loss": 0.10558454692363739, "num_tokens": 3810641.0, "reward": 0.3960937559604645, "reward_std": 0.5435082316398621, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.4439062476158142, "rewards/length_penalty/std": 0.2379816770553589, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9925057888031006, "sampling/importance_sampling_ratio/min": 0.09733714908361435, "sampling/sampling_logp_difference/max": 2.3295745849609375, "sampling/sampling_logp_difference/mean": 0.02152923122048378, "step": 54, "step_time": 23.610432841815054 }, { "clip_ratio/high_max": 0.0008515585839631967, "clip_ratio/high_mean": 0.0008515585839631967, "clip_ratio/low_mean": 0.0001473895667004399, "clip_ratio/low_min": 0.0001473895667004399, "clip_ratio/region_mean": 0.0009989481506636367, "completions/clipped_ratio": 0.05999999865889549, "completions/max_length": 2048.0, "completions/max_terminated_length": 2034.0, "completions/mean_length": 980.7599487304688, "completions/mean_terminated_length": 912.6382446289062, "completions/min_length": 400.0, "completions/min_terminated_length": 400.0, "entropy": 0.2108856201171875, "epoch": 0.14945652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.018623875454068184, "learning_rate": 5e-05, "loss": 0.08376599848270416, "num_tokens": 3862569.0, "reward": 0.44111326336860657, "reward_std": 0.47280439734458923, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404752373695374, "rewards/length_penalty/mean": -0.4788867235183716, "rewards/length_penalty/std": 0.2681271433830261, "sampling/importance_sampling_ratio/max": 2.98014760017395, "sampling/importance_sampling_ratio/mean": 0.9922913312911987, "sampling/importance_sampling_ratio/min": 0.04121199995279312, "sampling/sampling_logp_difference/max": 3.18902587890625, "sampling/sampling_logp_difference/mean": 0.02175760082900524, "step": 55, "step_time": 23.80699737602845 }, { "clip_ratio/high_max": 0.0010450290283188224, "clip_ratio/high_mean": 0.0010450290283188224, "clip_ratio/low_mean": 7.015509763732552e-05, "clip_ratio/low_min": 7.015509763732552e-05, "clip_ratio/region_mean": 0.0011151841375976802, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 1254.0, "completions/mean_length": 845.5799560546875, "completions/mean_terminated_length": 544.9750366210938, "completions/min_length": 280.0, "completions/min_terminated_length": 280.0, "entropy": 0.181646828353405, "epoch": 0.15217391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.01523221842944622, "learning_rate": 5e-05, "loss": 0.03551851212978363, "num_tokens": 3907918.0, "reward": 0.18711914122104645, "reward_std": 0.7015714049339294, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.41288086771965027, "rewards/length_penalty/std": 0.31132909655570984, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9930831789970398, "sampling/importance_sampling_ratio/min": 0.014245419763028622, "sampling/sampling_logp_difference/max": 4.251319885253906, "sampling/sampling_logp_difference/mean": 0.020080706104636192, "step": 56, "step_time": 23.348601100733504 }, { "clip_ratio/high_max": 0.0014594922540709376, "clip_ratio/high_mean": 0.0014594922540709376, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014594922540709376, "completions/clipped_ratio": 0.19999998807907104, "completions/max_length": 2048.0, "completions/max_terminated_length": 2024.0, "completions/mean_length": 1000.7799682617188, "completions/mean_terminated_length": 738.9750366210938, "completions/min_length": 255.0, "completions/min_terminated_length": 255.0, "entropy": 0.21108851730823516, "epoch": 0.15489130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.015691101551055908, "learning_rate": 5e-05, "loss": 0.040988340973854065, "num_tokens": 3961197.0, "reward": 0.3113378882408142, "reward_std": 0.6955205798149109, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.48866212368011475, "rewards/length_penalty/std": 0.3343265652656555, "sampling/importance_sampling_ratio/max": 2.521744966506958, "sampling/importance_sampling_ratio/mean": 0.991180956363678, "sampling/importance_sampling_ratio/min": 0.06271397322416306, "sampling/sampling_logp_difference/max": 2.7691709995269775, "sampling/sampling_logp_difference/mean": 0.0215458907186985, "step": 57, "step_time": 24.776436982909217 }, { "clip_ratio/high_max": 0.0007497882703319192, "clip_ratio/high_mean": 0.0007497882703319192, "clip_ratio/low_mean": 0.0001229701971169561, "clip_ratio/low_min": 0.0001229701971169561, "clip_ratio/region_mean": 0.0008727584558073431, "completions/clipped_ratio": 0.03999999910593033, "completions/max_length": 2048.0, "completions/max_terminated_length": 1817.0, "completions/mean_length": 938.3599853515625, "completions/mean_terminated_length": 892.125, "completions/min_length": 367.0, "completions/min_terminated_length": 367.0, "entropy": 0.23515559434890748, "epoch": 0.15760869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.018419070169329643, "learning_rate": 5e-05, "loss": 0.08900219947099686, "num_tokens": 4011925.0, "reward": 0.28181639313697815, "reward_std": 0.5855302214622498, "rewards/correctness/mean": 0.7400000095367432, "rewards/correctness/std": 0.44308751821517944, "rewards/length_penalty/mean": -0.4581835865974426, "rewards/length_penalty/std": 0.24371984601020813, "sampling/importance_sampling_ratio/max": 2.5031044483184814, "sampling/importance_sampling_ratio/mean": 0.9909224510192871, "sampling/importance_sampling_ratio/min": 0.09938520193099976, "sampling/sampling_logp_difference/max": 2.3087520599365234, "sampling/sampling_logp_difference/mean": 0.02372322604060173, "step": 58, "step_time": 24.546692570205778 }, { "clip_ratio/high_max": 0.000753168307710439, "clip_ratio/high_mean": 0.000753168307710439, "clip_ratio/low_mean": 4.419889592099935e-05, "clip_ratio/low_min": 4.419889592099935e-05, "clip_ratio/region_mean": 0.0007973672007210552, "completions/clipped_ratio": 0.0, "completions/max_length": 1366.0, "completions/max_terminated_length": 1366.0, "completions/mean_length": 590.4199829101562, "completions/mean_terminated_length": 590.4199829101562, "completions/min_length": 321.0, "completions/min_terminated_length": 321.0, "entropy": 0.14277512580156326, "epoch": 0.16032608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.015130100771784782, "learning_rate": 5e-05, "loss": 0.047128088772296906, "num_tokens": 4044156.0, "reward": 0.7117089629173279, "reward_std": 0.1164940893650055, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.28829100728034973, "rewards/length_penalty/std": 0.11649409681558609, "sampling/importance_sampling_ratio/max": 2.882826328277588, "sampling/importance_sampling_ratio/mean": 0.9934148192405701, "sampling/importance_sampling_ratio/min": 0.03744896501302719, "sampling/sampling_logp_difference/max": 3.284776210784912, "sampling/sampling_logp_difference/mean": 0.021746011450886726, "step": 59, "step_time": 15.403940830612555 }, { "clip_ratio/high_max": 0.0005577924195677042, "clip_ratio/high_mean": 0.0005577924195677042, "clip_ratio/low_mean": 3.265839186497033e-05, "clip_ratio/low_min": 3.265839186497033e-05, "clip_ratio/region_mean": 0.0005904508056119084, "completions/clipped_ratio": 0.0, "completions/max_length": 966.0, "completions/max_terminated_length": 966.0, "completions/mean_length": 647.739990234375, "completions/mean_terminated_length": 647.739990234375, "completions/min_length": 273.0, "completions/min_terminated_length": 273.0, "entropy": 0.14415572881698607, "epoch": 0.16304347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.014020378701388836, "learning_rate": 5e-05, "loss": 0.037132516503334045, "num_tokens": 4079353.0, "reward": 0.023720702156424522, "reward_std": 0.48956042528152466, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.3162792921066284, "rewards/length_penalty/std": 0.09593462198972702, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9944519400596619, "sampling/importance_sampling_ratio/min": 0.1064625084400177, "sampling/sampling_logp_difference/max": 2.239962339401245, "sampling/sampling_logp_difference/mean": 0.02042674832046032, "step": 60, "step_time": 12.129048788920045 }, { "clip_ratio/high_max": 0.0005259863304672763, "clip_ratio/high_mean": 0.0005259863304672763, "clip_ratio/low_mean": 3.6643460043706e-05, "clip_ratio/low_min": 3.6643460043706e-05, "clip_ratio/region_mean": 0.0005626297905109822, "completions/clipped_ratio": 0.0, "completions/max_length": 792.0, "completions/max_terminated_length": 792.0, "completions/mean_length": 511.5999755859375, "completions/mean_terminated_length": 511.5999755859375, "completions/min_length": 359.0, "completions/min_terminated_length": 359.0, "entropy": 0.14185337126255035, "epoch": 0.16576086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.015215730294585228, "learning_rate": 5e-05, "loss": 0.03220529854297638, "num_tokens": 4107343.0, "reward": 0.750195324420929, "reward_std": 0.049423981457948685, "rewards/correctness/mean": 1.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.24980469048023224, "rewards/length_penalty/std": 0.049423981457948685, "sampling/importance_sampling_ratio/max": 2.673095226287842, "sampling/importance_sampling_ratio/mean": 0.9938417673110962, "sampling/importance_sampling_ratio/min": 0.14251725375652313, "sampling/sampling_logp_difference/max": 1.9482922554016113, "sampling/sampling_logp_difference/mean": 0.02224837988615036, "step": 61, "step_time": 10.15479805925861 }, { "clip_ratio/high_max": 0.0005749615869717672, "clip_ratio/high_mean": 0.0005749615869717672, "clip_ratio/low_mean": 2.9107843874953688e-05, "clip_ratio/low_min": 2.9107843874953688e-05, "clip_ratio/region_mean": 0.000604069430846721, "completions/clipped_ratio": 0.0, "completions/max_length": 1501.0, "completions/max_terminated_length": 1501.0, "completions/mean_length": 755.9599609375, "completions/mean_terminated_length": 755.9599609375, "completions/min_length": 234.0, "completions/min_terminated_length": 234.0, "entropy": 0.13990469127893448, "epoch": 0.16847826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.015615438111126423, "learning_rate": 5e-05, "loss": 0.06095185503363609, "num_tokens": 4148761.0, "reward": 0.4708788990974426, "reward_std": 0.4449301064014435, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.36912110447883606, "rewards/length_penalty/std": 0.16295446455478668, "sampling/importance_sampling_ratio/max": 2.4815144538879395, "sampling/importance_sampling_ratio/mean": 0.9948198795318604, "sampling/importance_sampling_ratio/min": 0.14511510729789734, "sampling/sampling_logp_difference/max": 1.9302279949188232, "sampling/sampling_logp_difference/mean": 0.01760508492588997, "step": 62, "step_time": 17.662152016768232 }, { "clip_ratio/high_max": 0.00030208398529794065, "clip_ratio/high_mean": 0.00030208398529794065, "clip_ratio/low_mean": 7.888693944551051e-05, "clip_ratio/low_min": 7.888693944551051e-05, "clip_ratio/region_mean": 0.0003809709247434512, "completions/clipped_ratio": 0.0, "completions/max_length": 1402.0, "completions/max_terminated_length": 1402.0, "completions/mean_length": 583.3599853515625, "completions/mean_terminated_length": 583.3599853515625, "completions/min_length": 204.0, "completions/min_terminated_length": 204.0, "entropy": 0.16161190271377562, "epoch": 0.17119565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.013833782635629177, "learning_rate": 5e-05, "loss": 0.03906881809234619, "num_tokens": 4181179.0, "reward": 0.5151562094688416, "reward_std": 0.5463297963142395, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.2848437428474426, "rewards/length_penalty/std": 0.16151489317417145, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9928602576255798, "sampling/importance_sampling_ratio/min": 0.17339856922626495, "sampling/sampling_logp_difference/max": 1.7521624565124512, "sampling/sampling_logp_difference/mean": 0.022924896329641342, "step": 63, "step_time": 16.010359245119616 }, { "clip_ratio/high_max": 0.0006868168682558462, "clip_ratio/high_mean": 0.0006868168682558462, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0006868168682558462, "completions/clipped_ratio": 0.0, "completions/max_length": 904.0, "completions/max_terminated_length": 904.0, "completions/mean_length": 568.6799926757812, "completions/mean_terminated_length": 568.6799926757812, "completions/min_length": 342.0, "completions/min_terminated_length": 342.0, "entropy": 0.14348321557044982, "epoch": 0.17391304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.014510437846183777, "learning_rate": 5e-05, "loss": 0.024548858404159546, "num_tokens": 4213133.0, "reward": 0.682324230670929, "reward_std": 0.21207433938980103, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.2776757776737213, "rewards/length_penalty/std": 0.06843211501836777, "sampling/importance_sampling_ratio/max": 2.44184947013855, "sampling/importance_sampling_ratio/mean": 0.9942808747291565, "sampling/importance_sampling_ratio/min": 0.09248637408018112, "sampling/sampling_logp_difference/max": 2.3806939125061035, "sampling/sampling_logp_difference/mean": 0.021610219031572342, "step": 64, "step_time": 10.921900424407795 }, { "clip_ratio/high_max": 0.0010910295764915644, "clip_ratio/high_mean": 0.0010910295764915644, "clip_ratio/low_mean": 6.204147648531943e-05, "clip_ratio/low_min": 6.204147648531943e-05, "clip_ratio/region_mean": 0.001153071061708033, "completions/clipped_ratio": 0.0, "completions/max_length": 1477.0, "completions/max_terminated_length": 1477.0, "completions/mean_length": 632.5799560546875, "completions/mean_terminated_length": 632.5799560546875, "completions/min_length": 228.0, "completions/min_terminated_length": 228.0, "entropy": 0.17525759637355803, "epoch": 0.1766304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.020639101043343544, "learning_rate": 5e-05, "loss": 0.05388612672686577, "num_tokens": 4248472.0, "reward": 0.651123046875, "reward_std": 0.2796420753002167, "rewards/correctness/mean": 0.9599999785423279, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.30887696146965027, "rewards/length_penalty/std": 0.13174009323120117, "sampling/importance_sampling_ratio/max": 2.918149948120117, "sampling/importance_sampling_ratio/mean": 0.9931374788284302, "sampling/importance_sampling_ratio/min": 0.13037054240703583, "sampling/sampling_logp_difference/max": 2.037374496459961, "sampling/sampling_logp_difference/mean": 0.023967141285538673, "step": 65, "step_time": 17.327877108938992 }, { "clip_ratio/high_max": 0.001119992573512718, "clip_ratio/high_mean": 0.001119992573512718, "clip_ratio/low_mean": 0.00020487242727540433, "clip_ratio/low_min": 0.00020487242727540433, "clip_ratio/region_mean": 0.0013248650124296547, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1869.0, "completions/mean_length": 654.1400146484375, "completions/mean_terminated_length": 499.2666931152344, "completions/min_length": 214.0, "completions/min_terminated_length": 214.0, "entropy": 0.2962186366319656, "epoch": 0.1793478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.017421096563339233, "learning_rate": 5e-05, "loss": 0.04589683935046196, "num_tokens": 4284229.0, "reward": 0.44059568643569946, "reward_std": 0.6957334876060486, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.3194043040275574, "rewards/length_penalty/std": 0.288326621055603, "sampling/importance_sampling_ratio/max": 2.876610040664673, "sampling/importance_sampling_ratio/mean": 0.9876154065132141, "sampling/importance_sampling_ratio/min": 0.1718895584344864, "sampling/sampling_logp_difference/max": 1.7609031200408936, "sampling/sampling_logp_difference/mean": 0.03127918392419815, "step": 66, "step_time": 22.728930801153183 }, { "clip_ratio/high_max": 0.0003629518178058788, "clip_ratio/high_mean": 0.0003629518178058788, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0003629518178058788, "completions/clipped_ratio": 0.0, "completions/max_length": 894.0, "completions/max_terminated_length": 894.0, "completions/mean_length": 530.0599975585938, "completions/mean_terminated_length": 530.0599975585938, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.12096002399921417, "epoch": 0.18206521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.012911823578178883, "learning_rate": 5e-05, "loss": 0.032062433660030365, "num_tokens": 4313682.0, "reward": 0.3411816358566284, "reward_std": 0.5327054262161255, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.2588183581829071, "rewards/length_penalty/std": 0.09697854518890381, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9948055744171143, "sampling/importance_sampling_ratio/min": 0.23758108913898468, "sampling/sampling_logp_difference/max": 1.437246322631836, "sampling/sampling_logp_difference/mean": 0.020097574219107628, "step": 67, "step_time": 10.677236109972 }, { "clip_ratio/high_max": 0.0005797879653982818, "clip_ratio/high_mean": 0.0005797879653982818, "clip_ratio/low_mean": 0.00017863708490040153, "clip_ratio/low_min": 0.00017863708490040153, "clip_ratio/region_mean": 0.0007584250532090663, "completions/clipped_ratio": 0.0, "completions/max_length": 1754.0, "completions/max_terminated_length": 1754.0, "completions/mean_length": 629.3599853515625, "completions/mean_terminated_length": 629.3599853515625, "completions/min_length": 312.0, "completions/min_terminated_length": 312.0, "entropy": 0.15794651806354523, "epoch": 0.18478260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.01505596935749054, "learning_rate": 5e-05, "loss": 0.04626861959695816, "num_tokens": 4348910.0, "reward": 0.4526953101158142, "reward_std": 0.5702562928199768, "rewards/correctness/mean": 0.7599999904632568, "rewards/correctness/std": 0.43141910433769226, "rewards/length_penalty/mean": -0.3073046803474426, "rewards/length_penalty/std": 0.16258111596107483, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.994085967540741, "sampling/importance_sampling_ratio/min": 0.2601889669895172, "sampling/sampling_logp_difference/max": 1.3463470935821533, "sampling/sampling_logp_difference/mean": 0.02229192666709423, "step": 68, "step_time": 19.23433845723048 }, { "clip_ratio/high_max": 0.0009082971897441894, "clip_ratio/high_mean": 0.0009082971897441894, "clip_ratio/low_mean": 0.00014585773169528693, "clip_ratio/low_min": 0.00014585773169528693, "clip_ratio/region_mean": 0.0010541549359913915, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1624.0, "completions/mean_length": 494.41998291015625, "completions/mean_terminated_length": 462.7142639160156, "completions/min_length": 242.0, "completions/min_terminated_length": 242.0, "entropy": 0.2148962765932083, "epoch": 0.1875, "frac_reward_zero_std": 0.0, "grad_norm": 0.018317608162760735, "learning_rate": 5e-05, "loss": 0.054631538689136505, "num_tokens": 4376011.0, "reward": 0.3585839867591858, "reward_std": 0.6195550560951233, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.24141602218151093, "rewards/length_penalty/std": 0.1919320970773697, "sampling/importance_sampling_ratio/max": 2.9950358867645264, "sampling/importance_sampling_ratio/mean": 0.9916190505027771, "sampling/importance_sampling_ratio/min": 0.2629293203353882, "sampling/sampling_logp_difference/max": 1.3358700275421143, "sampling/sampling_logp_difference/mean": 0.027398645877838135, "step": 69, "step_time": 22.091090407921 }, { "clip_ratio/high_max": 0.0006563576433109119, "clip_ratio/high_mean": 0.0006563576433109119, "clip_ratio/low_mean": 0.00010137984645552933, "clip_ratio/low_min": 0.00010137984645552933, "clip_ratio/region_mean": 0.0007577374868560583, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 1427.0, "completions/mean_length": 679.5, "completions/mean_terminated_length": 527.4444580078125, "completions/min_length": 172.0, "completions/min_terminated_length": 172.0, "entropy": 0.1653291940689087, "epoch": 0.19021739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.018064742907881737, "learning_rate": 5e-05, "loss": 0.12476000189781189, "num_tokens": 4413496.0, "reward": -0.09178710728883743, "reward_std": 0.5935255289077759, "rewards/correctness/mean": 0.23999999463558197, "rewards/correctness/std": 0.43141913414001465, "rewards/length_penalty/mean": -0.331787109375, "rewards/length_penalty/std": 0.28031834959983826, "sampling/importance_sampling_ratio/max": 2.4027974605560303, "sampling/importance_sampling_ratio/mean": 0.9935649037361145, "sampling/importance_sampling_ratio/min": 0.3198908567428589, "sampling/sampling_logp_difference/max": 1.1397753953933716, "sampling/sampling_logp_difference/mean": 0.01983560621738434, "step": 70, "step_time": 22.78736790176481 }, { "clip_ratio/high_max": 0.0013086311635561287, "clip_ratio/high_mean": 0.0013086311635561287, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0013086311635561287, "completions/clipped_ratio": 0.0, "completions/max_length": 1391.0, "completions/max_terminated_length": 1391.0, "completions/mean_length": 308.55999755859375, "completions/mean_terminated_length": 308.55999755859375, "completions/min_length": 196.0, "completions/min_terminated_length": 196.0, "entropy": 0.1898427575826645, "epoch": 0.19293478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.016043968498706818, "learning_rate": 5e-05, "loss": 0.0421726331114769, "num_tokens": 4431284.0, "reward": 0.4493359327316284, "reward_std": 0.5210909843444824, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.1506640613079071, "rewards/length_penalty/std": 0.08385021239519119, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9918829798698425, "sampling/importance_sampling_ratio/min": 0.2813933491706848, "sampling/sampling_logp_difference/max": 1.2680017948150635, "sampling/sampling_logp_difference/mean": 0.03239607810974121, "step": 71, "step_time": 13.94420051225461 }, { "clip_ratio/high_max": 0.0007518933329265564, "clip_ratio/high_mean": 0.0007518933329265564, "clip_ratio/low_mean": 0.00022899623145349323, "clip_ratio/low_min": 0.00022899623145349323, "clip_ratio/region_mean": 0.0009808895352762192, "completions/clipped_ratio": 0.0, "completions/max_length": 649.0, "completions/max_terminated_length": 649.0, "completions/mean_length": 341.1000061035156, "completions/mean_terminated_length": 341.1000061035156, "completions/min_length": 132.0, "completions/min_terminated_length": 132.0, "entropy": 0.1368246629834175, "epoch": 0.1956521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015212559141218662, "learning_rate": 5e-05, "loss": 0.01818622462451458, "num_tokens": 4452009.0, "reward": 0.6334472298622131, "reward_std": 0.45945945382118225, "rewards/correctness/mean": 0.800000011920929, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.16655273735523224, "rewards/length_penalty/std": 0.0641215592622757, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9949499368667603, "sampling/importance_sampling_ratio/min": 0.25272005796432495, "sampling/sampling_logp_difference/max": 1.375472903251648, "sampling/sampling_logp_difference/mean": 0.02513076737523079, "step": 72, "step_time": 7.736045530997217 }, { "clip_ratio/high_max": 0.0005255058349575847, "clip_ratio/high_mean": 0.0005255058349575847, "clip_ratio/low_mean": 0.00011601120640989394, "clip_ratio/low_min": 0.00011601120640989394, "clip_ratio/region_mean": 0.0006415170442778617, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 1659.0, "completions/mean_length": 539.8399658203125, "completions/mean_terminated_length": 509.06121826171875, "completions/min_length": 184.0, "completions/min_terminated_length": 184.0, "entropy": 0.23647001385688782, "epoch": 0.1983695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.01909068040549755, "learning_rate": 5e-05, "loss": 0.047122035175561905, "num_tokens": 4481411.0, "reward": 0.35640624165534973, "reward_std": 0.6255202889442444, "rewards/correctness/mean": 0.6200000047683716, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.26359376311302185, "rewards/length_penalty/std": 0.21879027783870697, "sampling/importance_sampling_ratio/max": 2.6669154167175293, "sampling/importance_sampling_ratio/mean": 0.9911677241325378, "sampling/importance_sampling_ratio/min": 0.2946210503578186, "sampling/sampling_logp_difference/max": 1.2220653295516968, "sampling/sampling_logp_difference/mean": 0.027892595157027245, "step": 73, "step_time": 21.785567599115893 }, { "clip_ratio/high_max": 0.0010778171592392028, "clip_ratio/high_mean": 0.0010778171592392028, "clip_ratio/low_mean": 0.00011777303298003972, "clip_ratio/low_min": 0.00011777303298003972, "clip_ratio/region_mean": 0.0011955901863984764, "completions/clipped_ratio": 0.0, "completions/max_length": 554.0, "completions/max_terminated_length": 554.0, "completions/mean_length": 336.6600036621094, "completions/mean_terminated_length": 336.6600036621094, "completions/min_length": 208.0, "completions/min_terminated_length": 208.0, "entropy": 0.1645127385854721, "epoch": 0.20108695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.014695237390697002, "learning_rate": 5e-05, "loss": 0.013919752091169357, "num_tokens": 4501574.0, "reward": 0.17561523616313934, "reward_std": 0.4924038350582123, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.16438476741313934, "rewards/length_penalty/std": 0.04915780946612358, "sampling/importance_sampling_ratio/max": 2.9180831909179688, "sampling/importance_sampling_ratio/mean": 0.9932974576950073, "sampling/importance_sampling_ratio/min": 0.17829707264900208, "sampling/sampling_logp_difference/max": 1.72430419921875, "sampling/sampling_logp_difference/mean": 0.027354558929800987, "step": 74, "step_time": 6.898873920086771 }, { "clip_ratio/high_max": 0.0006365227804053575, "clip_ratio/high_mean": 0.0006365227804053575, "clip_ratio/low_mean": 0.00017117180395871402, "clip_ratio/low_min": 0.00017117180395871402, "clip_ratio/region_mean": 0.0008076945843640715, "completions/clipped_ratio": 0.0, "completions/max_length": 830.0, "completions/max_terminated_length": 830.0, "completions/mean_length": 281.29998779296875, "completions/mean_terminated_length": 281.29998779296875, "completions/min_length": 78.0, "completions/min_terminated_length": 78.0, "entropy": 0.16639892160892486, "epoch": 0.20380434782608695, "frac_reward_zero_std": 0.0, "grad_norm": 0.013990022242069244, "learning_rate": 5e-05, "loss": 0.01809602603316307, "num_tokens": 4517969.0, "reward": 0.7826464772224426, "reward_std": 0.3250719904899597, "rewards/correctness/mean": 0.9200000166893005, "rewards/correctness/std": 0.27404752373695374, "rewards/length_penalty/mean": -0.13735350966453552, "rewards/length_penalty/std": 0.10082448273897171, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9933053851127625, "sampling/importance_sampling_ratio/min": 0.18476475775241852, "sampling/sampling_logp_difference/max": 1.6886718273162842, "sampling/sampling_logp_difference/mean": 0.03111741691827774, "step": 75, "step_time": 9.629901089007035 }, { "clip_ratio/high_max": 0.0011410323728341608, "clip_ratio/high_mean": 0.0011410323728341608, "clip_ratio/low_mean": 0.00023014071630313992, "clip_ratio/low_min": 0.00023014071630313992, "clip_ratio/region_mean": 0.0013711730716750026, "completions/clipped_ratio": 0.0, "completions/max_length": 1694.0, "completions/max_terminated_length": 1694.0, "completions/mean_length": 380.8599853515625, "completions/mean_terminated_length": 380.8599853515625, "completions/min_length": 96.0, "completions/min_terminated_length": 96.0, "entropy": 0.23068214058876038, "epoch": 0.20652173913043478, "frac_reward_zero_std": 0.0, "grad_norm": 0.015038799494504929, "learning_rate": 5e-05, "loss": 0.05939723551273346, "num_tokens": 4540352.0, "reward": 0.5340331792831421, "reward_std": 0.6067567467689514, "rewards/correctness/mean": 0.7200000286102295, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.18596678972244263, "rewards/length_penalty/std": 0.20117054879665375, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9890738725662231, "sampling/importance_sampling_ratio/min": 0.20543357729911804, "sampling/sampling_logp_difference/max": 1.5826325416564941, "sampling/sampling_logp_difference/mean": 0.0316162109375, "step": 76, "step_time": 17.975655334303156 }, { "clip_ratio/high_max": 0.000963837094604969, "clip_ratio/high_mean": 0.000963837094604969, "clip_ratio/low_mean": 8.343763183802367e-05, "clip_ratio/low_min": 8.343763183802367e-05, "clip_ratio/region_mean": 0.0010472747264429927, "completions/clipped_ratio": 0.0, "completions/max_length": 660.0, "completions/max_terminated_length": 660.0, "completions/mean_length": 225.45999145507812, "completions/mean_terminated_length": 225.45999145507812, "completions/min_length": 73.0, "completions/min_terminated_length": 73.0, "entropy": 0.24213099777698516, "epoch": 0.20923913043478262, "frac_reward_zero_std": 0.0, "grad_norm": 0.01661447063088417, "learning_rate": 5e-05, "loss": 0.022843018174171448, "num_tokens": 4554185.0, "reward": 0.7299121022224426, "reward_std": 0.3942784070968628, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.11008789390325546, "rewards/length_penalty/std": 0.06080838665366173, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9897158145904541, "sampling/importance_sampling_ratio/min": 0.08884193003177643, "sampling/sampling_logp_difference/max": 2.420896530151367, "sampling/sampling_logp_difference/mean": 0.036450956016778946, "step": 77, "step_time": 7.283537532668561 }, { "clip_ratio/high_max": 0.0010524021228775382, "clip_ratio/high_mean": 0.0010524021228775382, "clip_ratio/low_mean": 0.00032099984819069507, "clip_ratio/low_min": 0.00032099984819069507, "clip_ratio/region_mean": 0.0013734019827097653, "completions/clipped_ratio": 0.0, "completions/max_length": 701.0, "completions/max_terminated_length": 701.0, "completions/mean_length": 230.0800018310547, "completions/mean_terminated_length": 230.0800018310547, "completions/min_length": 43.0, "completions/min_terminated_length": 43.0, "entropy": 0.1487855941057205, "epoch": 0.21195652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.017303554341197014, "learning_rate": 5e-05, "loss": 0.026571575552225113, "num_tokens": 4567989.0, "reward": 0.48765623569488525, "reward_std": 0.5217822790145874, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.11234375089406967, "rewards/length_penalty/std": 0.05492284148931503, "sampling/importance_sampling_ratio/max": 2.5257208347320557, "sampling/importance_sampling_ratio/mean": 0.9918290972709656, "sampling/importance_sampling_ratio/min": 0.22379803657531738, "sampling/sampling_logp_difference/max": 1.4970113039016724, "sampling/sampling_logp_difference/mean": 0.03053290583193302, "step": 78, "step_time": 7.628504432039335 }, { "clip_ratio/high_max": 0.0009887094842270018, "clip_ratio/high_mean": 0.0009887094842270018, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0009887094842270018, "completions/clipped_ratio": 0.0, "completions/max_length": 368.0, "completions/max_terminated_length": 368.0, "completions/mean_length": 226.0399932861328, "completions/mean_terminated_length": 226.0399932861328, "completions/min_length": 121.0, "completions/min_terminated_length": 121.0, "entropy": 0.17935218811035156, "epoch": 0.21467391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.015495066531002522, "learning_rate": 5e-05, "loss": 0.02079535648226738, "num_tokens": 4582751.0, "reward": 0.5896288752555847, "reward_std": 0.46082547307014465, "rewards/correctness/mean": 0.699999988079071, "rewards/correctness/std": 0.4629100263118744, "rewards/length_penalty/mean": -0.11037109047174454, "rewards/length_penalty/std": 0.03310258686542511, "sampling/importance_sampling_ratio/max": 2.6794304847717285, "sampling/importance_sampling_ratio/mean": 0.9911896586418152, "sampling/importance_sampling_ratio/min": 0.21863079071044922, "sampling/sampling_logp_difference/max": 1.5203709602355957, "sampling/sampling_logp_difference/mean": 0.03470217064023018, "step": 79, "step_time": 4.851263927994296 }, { "clip_ratio/high_max": 0.0008154305629432202, "clip_ratio/high_mean": 0.0008154305629432202, "clip_ratio/low_mean": 0.00045978412963449956, "clip_ratio/low_min": 0.00045978412963449956, "clip_ratio/region_mean": 0.001275214715860784, "completions/clipped_ratio": 0.0, "completions/max_length": 368.0, "completions/max_terminated_length": 368.0, "completions/mean_length": 160.3199920654297, "completions/mean_terminated_length": 160.3199920654297, "completions/min_length": 58.0, "completions/min_terminated_length": 58.0, "entropy": 0.23531386256217957, "epoch": 0.21739130434782608, "frac_reward_zero_std": 0.0, "grad_norm": 0.015668658539652824, "learning_rate": 5e-05, "loss": 0.017015229910612106, "num_tokens": 4593927.0, "reward": 0.3217187523841858, "reward_std": 0.504433810710907, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.07828125357627869, "rewards/length_penalty/std": 0.028255369514226913, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9909230470657349, "sampling/importance_sampling_ratio/min": 0.14803481101989746, "sampling/sampling_logp_difference/max": 1.9103078842163086, "sampling/sampling_logp_difference/mean": 0.039929430931806564, "step": 80, "step_time": 4.508802163414657 }, { "clip_ratio/high_max": 0.0010360270738601685, "clip_ratio/high_mean": 0.0010360270738601685, "clip_ratio/low_mean": 0.0004222874063998461, "clip_ratio/low_min": 0.0004222874063998461, "clip_ratio/region_mean": 0.001458314445335418, "completions/clipped_ratio": 0.0, "completions/max_length": 292.0, "completions/max_terminated_length": 292.0, "completions/mean_length": 157.0800018310547, "completions/mean_terminated_length": 157.0800018310547, "completions/min_length": 34.0, "completions/min_terminated_length": 34.0, "entropy": 0.2283316969871521, "epoch": 0.22010869565217392, "frac_reward_zero_std": 0.0, "grad_norm": 0.01512504555284977, "learning_rate": 5e-05, "loss": 0.015309778973460197, "num_tokens": 4606241.0, "reward": 0.4833007752895355, "reward_std": 0.49084603786468506, "rewards/correctness/mean": 0.5600000023841858, "rewards/correctness/std": 0.5014265179634094, "rewards/length_penalty/mean": -0.07669921964406967, "rewards/length_penalty/std": 0.03457314893603325, "sampling/importance_sampling_ratio/max": 2.6002397537231445, "sampling/importance_sampling_ratio/mean": 0.9895054697990417, "sampling/importance_sampling_ratio/min": 0.0006602171342819929, "sampling/sampling_logp_difference/max": 7.322941780090332, "sampling/sampling_logp_difference/mean": 0.039938464760780334, "step": 81, "step_time": 4.207236580783501 }, { "clip_ratio/high_max": 0.002563599334098399, "clip_ratio/high_mean": 0.002563599334098399, "clip_ratio/low_mean": 0.001496383035555482, "clip_ratio/low_min": 0.001496383035555482, "clip_ratio/region_mean": 0.004059982346370816, "completions/clipped_ratio": 0.0, "completions/max_length": 195.0, "completions/max_terminated_length": 195.0, "completions/mean_length": 97.41999816894531, "completions/mean_terminated_length": 97.41999816894531, "completions/min_length": 40.0, "completions/min_terminated_length": 40.0, "entropy": 0.22219233810901642, "epoch": 0.22282608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.014111941680312157, "learning_rate": 5e-05, "loss": 0.007607590407133102, "num_tokens": 4613962.0, "reward": 0.5524316430091858, "reward_std": 0.5043026208877563, "rewards/correctness/mean": 0.6000000238418579, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.04756835848093033, "rewards/length_penalty/std": 0.019571339711546898, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.990696907043457, "sampling/importance_sampling_ratio/min": 0.23683501780033112, "sampling/sampling_logp_difference/max": 1.4403915405273438, "sampling/sampling_logp_difference/mean": 0.03997170552611351, "step": 82, "step_time": 2.764910092111677 }, { "clip_ratio/high_max": 0.001733548124320805, "clip_ratio/high_mean": 0.001733548124320805, "clip_ratio/low_mean": 0.0007261270890012384, "clip_ratio/low_min": 0.0007261270890012384, "clip_ratio/region_mean": 0.0024596752366051076, "completions/clipped_ratio": 0.0, "completions/max_length": 146.0, "completions/max_terminated_length": 146.0, "completions/mean_length": 60.81999969482422, "completions/mean_terminated_length": 60.81999969482422, "completions/min_length": 25.0, "completions/min_terminated_length": 25.0, "entropy": 0.22290079295635223, "epoch": 0.22554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.013176962733268738, "learning_rate": 5e-05, "loss": 0.008476165123283863, "num_tokens": 4619433.0, "reward": 0.3703027367591858, "reward_std": 0.4969017505645752, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.4948716461658478, "rewards/length_penalty/mean": -0.029697265475988388, "rewards/length_penalty/std": 0.013850324787199497, "sampling/importance_sampling_ratio/max": 2.736015558242798, "sampling/importance_sampling_ratio/mean": 0.9942981004714966, "sampling/importance_sampling_ratio/min": 0.19088254868984222, "sampling/sampling_logp_difference/max": 1.6560969352722168, "sampling/sampling_logp_difference/mean": 0.041044432669878006, "step": 83, "step_time": 2.8419215080793947 }, { "clip_ratio/high_max": 0.001774064335040748, "clip_ratio/high_mean": 0.001774064335040748, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001774064335040748, "completions/clipped_ratio": 0.0, "completions/max_length": 434.0, "completions/max_terminated_length": 434.0, "completions/mean_length": 102.54000091552734, "completions/mean_terminated_length": 102.54000091552734, "completions/min_length": 26.0, "completions/min_terminated_length": 26.0, "entropy": 0.34105626940727235, "epoch": 0.22826086956521738, "frac_reward_zero_std": 0.0, "grad_norm": 0.01473130565136671, "learning_rate": 5e-05, "loss": 0.011427721939980984, "num_tokens": 4627540.0, "reward": 0.44993162155151367, "reward_std": 0.5254913568496704, "rewards/correctness/mean": 0.5, "rewards/correctness/std": 0.5050762891769409, "rewards/length_penalty/mean": -0.050068359822034836, "rewards/length_penalty/std": 0.049342118203639984, "sampling/importance_sampling_ratio/max": 2.824638605117798, "sampling/importance_sampling_ratio/mean": 0.9884728789329529, "sampling/importance_sampling_ratio/min": 0.24390774965286255, "sampling/sampling_logp_difference/max": 1.4109652042388916, "sampling/sampling_logp_difference/mean": 0.043421175330877304, "step": 84, "step_time": 5.026303122052923 }, { "clip_ratio/high_max": 0.0021632393356412647, "clip_ratio/high_mean": 0.0021632393356412647, "clip_ratio/low_mean": 0.0007204134250059724, "clip_ratio/low_min": 0.0007204134250059724, "clip_ratio/region_mean": 0.0028836527839303018, "completions/clipped_ratio": 0.0, "completions/max_length": 402.0, "completions/max_terminated_length": 402.0, "completions/mean_length": 80.37999725341797, "completions/mean_terminated_length": 80.37999725341797, "completions/min_length": 29.0, "completions/min_terminated_length": 29.0, "entropy": 0.30774680972099305, "epoch": 0.23097826086956522, "frac_reward_zero_std": 0.0, "grad_norm": 0.015826839953660965, "learning_rate": 5e-05, "loss": 0.018494833260774612, "num_tokens": 4633579.0, "reward": 0.3607519567012787, "reward_std": 0.5096510052680969, "rewards/correctness/mean": 0.4000000059604645, "rewards/correctness/std": 0.49487167596817017, "rewards/length_penalty/mean": -0.03924804553389549, "rewards/length_penalty/std": 0.032291851937770844, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9881213903427124, "sampling/importance_sampling_ratio/min": 0.10218166559934616, "sampling/sampling_logp_difference/max": 2.281002998352051, "sampling/sampling_logp_difference/mean": 0.04308116063475609, "step": 85, "step_time": 4.310805863235146 }, { "clip_ratio/high_max": 0.002094677370041609, "clip_ratio/high_mean": 0.002094677370041609, "clip_ratio/low_mean": 0.00037807184271514415, "clip_ratio/low_min": 0.00037807184271514415, "clip_ratio/region_mean": 0.002472749212756753, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 47.87999725341797, "completions/mean_terminated_length": 47.87999725341797, "completions/min_length": 19.0, "completions/min_terminated_length": 19.0, "entropy": 0.2662091016769409, "epoch": 0.23369565217391305, "frac_reward_zero_std": 0.0, "grad_norm": 0.014103682711720467, "learning_rate": 5e-05, "loss": 0.0051345182582736015, "num_tokens": 4638073.0, "reward": 0.8166210651397705, "reward_std": 0.3698119521141052, "rewards/correctness/mean": 0.8399999737739563, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.023378906771540642, "rewards/length_penalty/std": 0.007568047381937504, "sampling/importance_sampling_ratio/max": 2.7011165618896484, "sampling/importance_sampling_ratio/mean": 0.9879431128501892, "sampling/importance_sampling_ratio/min": 0.2666534185409546, "sampling/sampling_logp_difference/max": 1.321805477142334, "sampling/sampling_logp_difference/mean": 0.04919382184743881, "step": 86, "step_time": 1.8847677169833332 }, { "clip_ratio/high_max": 0.0021833667065948247, "clip_ratio/high_mean": 0.0021833667065948247, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021833667065948247, "completions/clipped_ratio": 0.0, "completions/max_length": 202.0, "completions/max_terminated_length": 202.0, "completions/mean_length": 55.41999816894531, "completions/mean_terminated_length": 55.41999816894531, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.2376786708831787, "epoch": 0.23641304347826086, "frac_reward_zero_std": 0.0, "grad_norm": 0.01361458096653223, "learning_rate": 5e-05, "loss": 0.010031810030341148, "num_tokens": 4643054.0, "reward": 0.31293943524360657, "reward_std": 0.48329460620880127, "rewards/correctness/mean": 0.3400000035762787, "rewards/correctness/std": 0.4785180985927582, "rewards/length_penalty/mean": -0.02706054598093033, "rewards/length_penalty/std": 0.023779436945915222, "sampling/importance_sampling_ratio/max": 2.4680721759796143, "sampling/importance_sampling_ratio/mean": 0.9916805028915405, "sampling/importance_sampling_ratio/min": 0.24595335125923157, "sampling/sampling_logp_difference/max": 1.4026134014129639, "sampling/sampling_logp_difference/mean": 0.040355995297431946, "step": 87, "step_time": 2.768871467327699 }, { "clip_ratio/high_max": 0.0021316193044185637, "clip_ratio/high_mean": 0.0021316193044185637, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0021316193044185637, "completions/clipped_ratio": 0.0, "completions/max_length": 119.0, "completions/max_terminated_length": 119.0, "completions/mean_length": 37.29999923706055, "completions/mean_terminated_length": 37.29999923706055, "completions/min_length": 15.0, "completions/min_terminated_length": 15.0, "entropy": 0.2463228553533554, "epoch": 0.2391304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.01799052767455578, "learning_rate": 5e-05, "loss": 0.006354824639856815, "num_tokens": 4647809.0, "reward": 0.44178709387779236, "reward_std": 0.5040752291679382, "rewards/correctness/mean": 0.46000000834465027, "rewards/correctness/std": 0.5034573674201965, "rewards/length_penalty/mean": -0.01821289025247097, "rewards/length_penalty/std": 0.011233825236558914, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9960753917694092, "sampling/importance_sampling_ratio/min": 0.28705650568008423, "sampling/sampling_logp_difference/max": 1.3409881591796875, "sampling/sampling_logp_difference/mean": 0.04679141193628311, "step": 88, "step_time": 2.111156525555998 }, { "clip_ratio/high_max": 0.002344606863334775, "clip_ratio/high_mean": 0.002344606863334775, "clip_ratio/low_mean": 0.0017502497415989638, "clip_ratio/low_min": 0.0017502497415989638, "clip_ratio/region_mean": 0.0040948566049337385, "completions/clipped_ratio": 0.0, "completions/max_length": 113.0, "completions/max_terminated_length": 113.0, "completions/mean_length": 36.47999954223633, "completions/mean_terminated_length": 36.47999954223633, "completions/min_length": 16.0, "completions/min_terminated_length": 16.0, "entropy": 0.2862419366836548, "epoch": 0.2418478260869565, "frac_reward_zero_std": 0.0, "grad_norm": 0.015066390857100487, "learning_rate": 5e-05, "loss": 0.006557346787303686, "num_tokens": 4653093.0, "reward": 0.022187499329447746, "reward_std": 0.19955237209796906, "rewards/correctness/mean": 0.03999999910593033, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.017812499776482582, "rewards/length_penalty/std": 0.010008363053202629, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9836841225624084, "sampling/importance_sampling_ratio/min": 0.30964648723602295, "sampling/sampling_logp_difference/max": 1.1723240613937378, "sampling/sampling_logp_difference/mean": 0.04838472604751587, "step": 89, "step_time": 2.0711093137506396 }, { "clip_ratio/high_max": 0.0004728132393211126, "clip_ratio/high_mean": 0.0004728132393211126, "clip_ratio/low_mean": 0.0003629764076322317, "clip_ratio/low_min": 0.0003629764076322317, "clip_ratio/region_mean": 0.0008357896469533444, "completions/clipped_ratio": 0.0, "completions/max_length": 161.0, "completions/max_terminated_length": 161.0, "completions/mean_length": 43.040000915527344, "completions/mean_terminated_length": 43.040000915527344, "completions/min_length": 11.0, "completions/min_terminated_length": 11.0, "entropy": 0.34693005681037903, "epoch": 0.24456521739130435, "frac_reward_zero_std": 0.0, "grad_norm": 0.015743518248200417, "learning_rate": 5e-05, "loss": 0.01085373293608427, "num_tokens": 4658275.0, "reward": 0.15898437798023224, "reward_std": 0.3826689124107361, "rewards/correctness/mean": 0.18000000715255737, "rewards/correctness/std": 0.3880879282951355, "rewards/length_penalty/mean": -0.021015625447034836, "rewards/length_penalty/std": 0.01949433609843254, "sampling/importance_sampling_ratio/max": 1.6567002534866333, "sampling/importance_sampling_ratio/mean": 0.9856835007667542, "sampling/importance_sampling_ratio/min": 0.19376817345619202, "sampling/sampling_logp_difference/max": 1.6410927772521973, "sampling/sampling_logp_difference/mean": 0.04058392345905304, "step": 90, "step_time": 2.403021330712363 }, { "clip_ratio/high_max": 0.003640749305486679, "clip_ratio/high_mean": 0.003640749305486679, "clip_ratio/low_mean": 0.002236307132989168, "clip_ratio/low_min": 0.002236307132989168, "clip_ratio/region_mean": 0.005877056438475847, "completions/clipped_ratio": 0.0, "completions/max_length": 48.0, "completions/max_terminated_length": 48.0, "completions/mean_length": 17.020000457763672, "completions/mean_terminated_length": 17.020000457763672, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.24630841612815857, "epoch": 0.24728260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.012663047760725021, "learning_rate": 5e-05, "loss": 0.003191206604242325, "num_tokens": 4664296.0, "reward": 0.27168944478034973, "reward_std": 0.45263922214508057, "rewards/correctness/mean": 0.2800000011920929, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.008310547098517418, "rewards/length_penalty/std": 0.004402263555675745, "sampling/importance_sampling_ratio/max": 1.9033176898956299, "sampling/importance_sampling_ratio/mean": 0.9900517463684082, "sampling/importance_sampling_ratio/min": 0.2030993402004242, "sampling/sampling_logp_difference/max": 1.5940600633621216, "sampling/sampling_logp_difference/mean": 0.042618442326784134, "step": 91, "step_time": 1.784956254530698 }, { "clip_ratio/high_max": 0.0030817064456641675, "clip_ratio/high_mean": 0.0030817064456641675, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0030817064456641675, "completions/clipped_ratio": 0.0, "completions/max_length": 53.0, "completions/max_terminated_length": 53.0, "completions/mean_length": 20.299999237060547, "completions/mean_terminated_length": 20.299999237060547, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.18247316479682923, "epoch": 0.25, "frac_reward_zero_std": 0.0, "grad_norm": 0.012025109492242336, "learning_rate": 5e-05, "loss": 0.0017745597288012505, "num_tokens": 4667701.0, "reward": 0.2700878977775574, "reward_std": 0.4487000107765198, "rewards/correctness/mean": 0.2800000011920929, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.00991210900247097, "rewards/length_penalty/std": 0.00615126034244895, "sampling/importance_sampling_ratio/max": 2.7393031120300293, "sampling/importance_sampling_ratio/mean": 0.992425799369812, "sampling/importance_sampling_ratio/min": 0.439604789018631, "sampling/sampling_logp_difference/max": 1.0077035427093506, "sampling/sampling_logp_difference/mean": 0.028682120144367218, "step": 92, "step_time": 1.6275623377878219 }, { "clip_ratio/high_max": 0.0033241380006074905, "clip_ratio/high_mean": 0.0033241380006074905, "clip_ratio/low_mean": 0.0003289473708719015, "clip_ratio/low_min": 0.0003289473708719015, "clip_ratio/region_mean": 0.003653085371479392, "completions/clipped_ratio": 0.0, "completions/max_length": 479.0, "completions/max_terminated_length": 479.0, "completions/mean_length": 21.100000381469727, "completions/mean_terminated_length": 21.100000381469727, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.24791947305202483, "epoch": 0.25271739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.01795797049999237, "learning_rate": 5e-05, "loss": 0.015818653628230095, "num_tokens": 4674376.0, "reward": 0.26969724893569946, "reward_std": 0.4579719305038452, "rewards/correctness/mean": 0.2800000011920929, "rewards/correctness/std": 0.4535573720932007, "rewards/length_penalty/mean": -0.010302734561264515, "rewards/length_penalty/std": 0.03234971687197685, "sampling/importance_sampling_ratio/max": 2.0742790699005127, "sampling/importance_sampling_ratio/mean": 0.9937615394592285, "sampling/importance_sampling_ratio/min": 0.406208336353302, "sampling/sampling_logp_difference/max": 0.9008891582489014, "sampling/sampling_logp_difference/mean": 0.02320687845349312, "step": 93, "step_time": 5.370015563676134 }, { "clip_ratio/high_max": 0.001408054633066058, "clip_ratio/high_mean": 0.001408054633066058, "clip_ratio/low_mean": 0.0011966381454840303, "clip_ratio/low_min": 0.0011966381454840303, "clip_ratio/region_mean": 0.002604692801833153, "completions/clipped_ratio": 0.0, "completions/max_length": 642.0, "completions/max_terminated_length": 642.0, "completions/mean_length": 37.97999954223633, "completions/mean_terminated_length": 37.97999954223633, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.3260998785495758, "epoch": 0.2554347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.017963992431759834, "learning_rate": 5e-05, "loss": 0.01967729814350605, "num_tokens": 4682305.0, "reward": 0.20145507156848907, "reward_std": 0.4240070581436157, "rewards/correctness/mean": 0.2199999988079071, "rewards/correctness/std": 0.4184519350528717, "rewards/length_penalty/mean": -0.018544921651482582, "rewards/length_penalty/std": 0.04375017434358597, "sampling/importance_sampling_ratio/max": 1.8460466861724854, "sampling/importance_sampling_ratio/mean": 0.9828484654426575, "sampling/importance_sampling_ratio/min": 0.209793820977211, "sampling/sampling_logp_difference/max": 1.5616300106048584, "sampling/sampling_logp_difference/mean": 0.04273902624845505, "step": 94, "step_time": 7.573918187059462 }, { "clip_ratio/high_max": 0.00416666679084301, "clip_ratio/high_mean": 0.00416666679084301, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00416666679084301, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 10.15999984741211, "completions/mean_terminated_length": 10.15999984741211, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.09587395042181016, "epoch": 0.25815217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.02545054256916046, "learning_rate": 5e-05, "loss": 0.0001122662506531924, "num_tokens": 4685183.0, "reward": 0.035039063543081284, "reward_std": 0.1979675143957138, "rewards/correctness/mean": 0.03999999910593033, "rewards/correctness/std": 0.1979486644268036, "rewards/length_penalty/mean": -0.004960937425494194, "rewards/length_penalty/std": 0.0010410021059215069, "sampling/importance_sampling_ratio/max": 1.9095886945724487, "sampling/importance_sampling_ratio/mean": 0.9855576157569885, "sampling/importance_sampling_ratio/min": 0.39807626605033875, "sampling/sampling_logp_difference/max": 0.9211117029190063, "sampling/sampling_logp_difference/mean": 0.03420029208064079, "step": 95, "step_time": 1.3936079579871148 }, { "clip_ratio/high_max": 0.0026901003904640673, "clip_ratio/high_mean": 0.0026901003904640673, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0026901003904640673, "completions/clipped_ratio": 0.0, "completions/max_length": 26.0, "completions/max_terminated_length": 26.0, "completions/mean_length": 14.779999732971191, "completions/mean_terminated_length": 14.779999732971191, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.07418197467923164, "epoch": 0.2608695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.013104759156703949, "learning_rate": 5e-05, "loss": 0.00014467601431533694, "num_tokens": 4689452.0, "reward": 0.352783203125, "reward_std": 0.48549649119377136, "rewards/correctness/mean": 0.36000001430511475, "rewards/correctness/std": 0.4848732054233551, "rewards/length_penalty/mean": -0.0072167967446148396, "rewards/length_penalty/std": 0.0024746423587203026, "sampling/importance_sampling_ratio/max": 2.37707781791687, "sampling/importance_sampling_ratio/mean": 0.9971712827682495, "sampling/importance_sampling_ratio/min": 0.507285475730896, "sampling/sampling_logp_difference/max": 0.8658719062805176, "sampling/sampling_logp_difference/mean": 0.011993075720965862, "step": 96, "step_time": 1.4969913428649306 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.001652892492711544, "clip_ratio/low_min": 0.001652892492711544, "clip_ratio/region_mean": 0.001652892492711544, "completions/clipped_ratio": 0.0, "completions/max_length": 23.0, "completions/max_terminated_length": 23.0, "completions/mean_length": 10.779999732971191, "completions/mean_terminated_length": 10.779999732971191, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.0697777010500431, "epoch": 0.26358695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.01211521029472351, "learning_rate": 5e-05, "loss": 0.0006838931585662067, "num_tokens": 4692921.0, "reward": 0.15473632514476776, "reward_std": 0.36963653564453125, "rewards/correctness/mean": 0.1599999964237213, "rewards/correctness/std": 0.37032803893089294, "rewards/length_penalty/mean": -0.0052636717446148396, "rewards/length_penalty/std": 0.001306067337282002, "sampling/importance_sampling_ratio/max": 1.4201691150665283, "sampling/importance_sampling_ratio/mean": 0.996337354183197, "sampling/importance_sampling_ratio/min": 0.31222423911094666, "sampling/sampling_logp_difference/max": 1.1640336513519287, "sampling/sampling_logp_difference/mean": 0.011246762238442898, "step": 97, "step_time": 1.446535994997248 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0017241379246115685, "clip_ratio/low_min": 0.0017241379246115685, "clip_ratio/region_mean": 0.0017241379246115685, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 11.119999885559082, "completions/mean_terminated_length": 11.119999885559082, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.06468469575047493, "epoch": 0.266304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.02177579142153263, "learning_rate": 5e-05, "loss": 0.00026727200020104647, "num_tokens": 4695617.0, "reward": 0.19457030296325684, "reward_std": 0.4038449823856354, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.0054296874441206455, "rewards/length_penalty/std": 0.0009287564898841083, "sampling/importance_sampling_ratio/max": 1.4296082258224487, "sampling/importance_sampling_ratio/mean": 0.9975820183753967, "sampling/importance_sampling_ratio/min": 0.7503319382667542, "sampling/sampling_logp_difference/max": 0.35740041732788086, "sampling/sampling_logp_difference/mean": 0.006841938477009535, "step": 98, "step_time": 1.4107645226176828 }, { "clip_ratio/high_max": 0.0014184396713972092, "clip_ratio/high_mean": 0.0014184396713972092, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0014184396713972092, "completions/clipped_ratio": 0.0, "completions/max_length": 31.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 13.34000015258789, "completions/mean_terminated_length": 13.34000015258789, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.09329047352075577, "epoch": 0.26902173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.019988512620329857, "learning_rate": 5e-05, "loss": 0.001125878538005054, "num_tokens": 4699194.0, "reward": 0.37348631024360657, "reward_std": 0.4906356930732727, "rewards/correctness/mean": 0.3799999952316284, "rewards/correctness/std": 0.4903143346309662, "rewards/length_penalty/mean": -0.006513671949505806, "rewards/length_penalty/std": 0.002664602128788829, "sampling/importance_sampling_ratio/max": 1.350227952003479, "sampling/importance_sampling_ratio/mean": 0.9988000988960266, "sampling/importance_sampling_ratio/min": 0.815227746963501, "sampling/sampling_logp_difference/max": 0.30027341842651367, "sampling/sampling_logp_difference/mean": 0.0107263857498765, "step": 99, "step_time": 1.525351885240525 }, { "clip_ratio/high_max": 0.001960784383118153, "clip_ratio/high_mean": 0.001960784383118153, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.001960784383118153, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 10.479999542236328, "completions/mean_terminated_length": 10.479999542236328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.03208514600992203, "epoch": 0.2717391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.014589188620448112, "learning_rate": 5e-05, "loss": 0.0002485829754732549, "num_tokens": 4701948.0, "reward": 0.1948828101158142, "reward_std": 0.404180109500885, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.005117187276482582, "rewards/length_penalty/std": 0.0007782548200339079, "sampling/importance_sampling_ratio/max": 1.0595662593841553, "sampling/importance_sampling_ratio/mean": 0.9968538880348206, "sampling/importance_sampling_ratio/min": 0.08549613505601883, "sampling/sampling_logp_difference/max": 2.4592840671539307, "sampling/sampling_logp_difference/mean": 0.008473201654851437, "step": 100, "step_time": 1.3946683751419187 }, { "clip_ratio/high_max": 0.0036949152126908302, "clip_ratio/high_mean": 0.0036949152126908302, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0036949152126908302, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 10.760000228881836, "completions/mean_terminated_length": 10.760000228881836, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.12038453966379166, "epoch": 0.27445652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.009302928112447262, "learning_rate": 5e-05, "loss": 0.0005147918127477169, "num_tokens": 4705376.0, "reward": 0.15474608540534973, "reward_std": 0.36920055747032166, "rewards/correctness/mean": 0.1599999964237213, "rewards/correctness/std": 0.37032806873321533, "rewards/length_penalty/mean": -0.005253906361758709, "rewards/length_penalty/std": 0.001276944880373776, "sampling/importance_sampling_ratio/max": 1.4610841274261475, "sampling/importance_sampling_ratio/mean": 0.9940187931060791, "sampling/importance_sampling_ratio/min": 0.5818896889686584, "sampling/sampling_logp_difference/max": 0.541474461555481, "sampling/sampling_logp_difference/mean": 0.01695821061730385, "step": 101, "step_time": 1.4081646401900798 }, { "clip_ratio/high_max": 0.0046176047995686535, "clip_ratio/high_mean": 0.0046176047995686535, "clip_ratio/low_mean": 0.002083333395421505, "clip_ratio/low_min": 0.002083333395421505, "clip_ratio/region_mean": 0.006700938194990158, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 12.479999542236328, "completions/mean_terminated_length": 12.479999542236328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.14907675981521606, "epoch": 0.27717391304347827, "frac_reward_zero_std": 0.0, "grad_norm": 0.02506999857723713, "learning_rate": 5e-05, "loss": 0.00030531908851116896, "num_tokens": 4708930.0, "reward": 0.1939062476158142, "reward_std": 0.4031943380832672, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.006093749776482582, "rewards/length_penalty/std": 0.0013748533092439175, "sampling/importance_sampling_ratio/max": 1.2105108499526978, "sampling/importance_sampling_ratio/mean": 0.9911222457885742, "sampling/importance_sampling_ratio/min": 0.3679906129837036, "sampling/sampling_logp_difference/max": 0.9996979236602783, "sampling/sampling_logp_difference/mean": 0.01580055058002472, "step": 102, "step_time": 1.4087490995880216 }, { "clip_ratio/high_max": 0.0016949152573943137, "clip_ratio/high_mean": 0.0016949152573943137, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0016949152573943137, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 11.079999923706055, "completions/mean_terminated_length": 11.079999923706055, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.1153342917561531, "epoch": 0.2798913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.011503961868584156, "learning_rate": 5e-05, "loss": 0.00048589514335617423, "num_tokens": 4713724.0, "reward": -0.005410156212747097, "reward_std": 0.0009086305508390069, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005410156212747097, "rewards/length_penalty/std": 0.000908630492631346, "sampling/importance_sampling_ratio/max": 1.4291812181472778, "sampling/importance_sampling_ratio/mean": 0.9990289211273193, "sampling/importance_sampling_ratio/min": 0.5646785497665405, "sampling/sampling_logp_difference/max": 0.5714986324310303, "sampling/sampling_logp_difference/mean": 0.01620861329138279, "step": 103, "step_time": 1.5233878679573536 }, { "clip_ratio/high_max": 0.0020618556067347527, "clip_ratio/high_mean": 0.0020618556067347527, "clip_ratio/low_mean": 0.001785714365541935, "clip_ratio/low_min": 0.001785714365541935, "clip_ratio/region_mean": 0.0038475699722766874, "completions/clipped_ratio": 0.0, "completions/max_length": 19.0, "completions/max_terminated_length": 19.0, "completions/mean_length": 10.300000190734863, "completions/mean_terminated_length": 10.300000190734863, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.10250563323497772, "epoch": 0.2826086956521739, "frac_reward_zero_std": 0.0, "grad_norm": 0.02407737821340561, "learning_rate": 5e-05, "loss": 0.000767544552218169, "num_tokens": 4717749.0, "reward": -0.0050292969681322575, "reward_std": 0.0013642305275425315, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0050292969681322575, "rewards/length_penalty/std": 0.0013642304111272097, "sampling/importance_sampling_ratio/max": 1.3338792324066162, "sampling/importance_sampling_ratio/mean": 0.9947230219841003, "sampling/importance_sampling_ratio/min": 0.7260318994522095, "sampling/sampling_logp_difference/max": 0.3201613426208496, "sampling/sampling_logp_difference/mean": 0.013192125596106052, "step": 104, "step_time": 1.4319615187123418 }, { "clip_ratio/high_max": 0.012666899152100087, "clip_ratio/high_mean": 0.012666899152100087, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.012666899152100087, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 11.479999542236328, "completions/mean_terminated_length": 11.479999542236328, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.09933862686157227, "epoch": 0.28532608695652173, "frac_reward_zero_std": 0.0, "grad_norm": 0.029227370396256447, "learning_rate": 5e-05, "loss": 0.0007443889626301825, "num_tokens": 4720683.0, "reward": -0.005605468526482582, "reward_std": 0.0014063441194593906, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005605468526482582, "rewards/length_penalty/std": 0.0014063441194593906, "sampling/importance_sampling_ratio/max": 1.9715588092803955, "sampling/importance_sampling_ratio/mean": 0.9908311367034912, "sampling/importance_sampling_ratio/min": 0.49639132618904114, "sampling/sampling_logp_difference/max": 0.7003906965255737, "sampling/sampling_logp_difference/mean": 0.020100245252251625, "step": 105, "step_time": 1.3756066388450563 }, { "clip_ratio/high_max": 0.00499330535531044, "clip_ratio/high_mean": 0.00499330535531044, "clip_ratio/low_mean": 0.0016260161995887757, "clip_ratio/low_min": 0.0016260161995887757, "clip_ratio/region_mean": 0.006619321554899216, "completions/clipped_ratio": 0.0, "completions/max_length": 22.0, "completions/max_terminated_length": 22.0, "completions/mean_length": 12.479999542236328, "completions/mean_terminated_length": 12.479999542236328, "completions/min_length": 10.0, "completions/min_terminated_length": 10.0, "entropy": 0.10761644840240478, "epoch": 0.28804347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.023589495569467545, "learning_rate": 5e-05, "loss": 0.000907458714209497, "num_tokens": 4723787.0, "reward": 0.05390624701976776, "reward_std": 0.23942075669765472, "rewards/correctness/mean": 0.05999999865889549, "rewards/correctness/std": 0.2398979514837265, "rewards/length_penalty/mean": -0.006093749776482582, "rewards/length_penalty/std": 0.001294654910452664, "sampling/importance_sampling_ratio/max": 1.7938042879104614, "sampling/importance_sampling_ratio/mean": 0.9942936301231384, "sampling/importance_sampling_ratio/min": 0.4382175803184509, "sampling/sampling_logp_difference/max": 0.8250397443771362, "sampling/sampling_logp_difference/mean": 0.026004480198025703, "step": 106, "step_time": 1.4649051453452557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 15.0, "completions/max_terminated_length": 15.0, "completions/mean_length": 9.399999618530273, "completions/mean_terminated_length": 9.399999618530273, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.061834586411714555, "epoch": 0.2907608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4728297.0, "reward": -0.004589843563735485, "reward_std": 0.0013810679083690047, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.004589843563735485, "rewards/length_penalty/std": 0.0013810679083690047, "sampling/importance_sampling_ratio/max": 2.081481695175171, "sampling/importance_sampling_ratio/mean": 0.9931551218032837, "sampling/importance_sampling_ratio/min": 0.5352470874786377, "sampling/sampling_logp_difference/max": 0.7330799102783203, "sampling/sampling_logp_difference/mean": 0.01708344742655754, "step": 107, "step_time": 1.4451532850507647 }, { "clip_ratio/high_max": 0.007184331864118576, "clip_ratio/high_mean": 0.007184331864118576, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007184331864118576, "completions/clipped_ratio": 0.0, "completions/max_length": 19.0, "completions/max_terminated_length": 19.0, "completions/mean_length": 11.359999656677246, "completions/mean_terminated_length": 11.359999656677246, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.13657148629426957, "epoch": 0.29347826086956524, "frac_reward_zero_std": 0.0, "grad_norm": 0.018689686432480812, "learning_rate": 5e-05, "loss": 0.0004315703408792615, "num_tokens": 4731915.0, "reward": -0.005546874832361937, "reward_std": 0.0017445869743824005, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.005546874832361937, "rewards/length_penalty/std": 0.0017445869743824005, "sampling/importance_sampling_ratio/max": 1.7193013429641724, "sampling/importance_sampling_ratio/mean": 0.9968085289001465, "sampling/importance_sampling_ratio/min": 0.2811293303966522, "sampling/sampling_logp_difference/max": 1.2689404487609863, "sampling/sampling_logp_difference/mean": 0.025670014321804047, "step": 108, "step_time": 1.9418589477427304 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 14.0, "completions/max_terminated_length": 14.0, "completions/mean_length": 9.59999942779541, "completions/mean_terminated_length": 9.59999942779541, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.08031524941325188, "epoch": 0.296195652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.029882347211241722, "learning_rate": 5e-05, "loss": 0.00019607065769378096, "num_tokens": 4734735.0, "reward": 0.01531249936670065, "reward_std": 0.1412566900253296, "rewards/correctness/mean": 0.019999999552965164, "rewards/correctness/std": 0.1414213627576828, "rewards/length_penalty/mean": -0.004687500186264515, "rewards/length_penalty/std": 0.0011333759175613523, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9957611560821533, "sampling/importance_sampling_ratio/min": 0.4287766218185425, "sampling/sampling_logp_difference/max": 2.0605242252349854, "sampling/sampling_logp_difference/mean": 0.03292733430862427, "step": 109, "step_time": 1.3846741961315274 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 12.0, "completions/max_terminated_length": 12.0, "completions/mean_length": 8.800000190734863, "completions/mean_terminated_length": 8.800000190734863, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.045162886008620264, "epoch": 0.29891304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4737755.0, "reward": -0.0042968750931322575, "reward_std": 0.0007891816785559058, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0042968750931322575, "rewards/length_penalty/std": 0.0007891816785559058, "sampling/importance_sampling_ratio/max": 2.4870872497558594, "sampling/importance_sampling_ratio/mean": 0.9982876777648926, "sampling/importance_sampling_ratio/min": 0.1595466136932373, "sampling/sampling_logp_difference/max": 1.8354191780090332, "sampling/sampling_logp_difference/mean": 0.02257714234292507, "step": 110, "step_time": 1.365794947836548 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 8.0, "completions/max_terminated_length": 8.0, "completions/mean_length": 8.0, "completions/mean_terminated_length": 8.0, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.04572306871414185, "epoch": 0.3016304347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4740815.0, "reward": -0.00390625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00390625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.153853416442871, "sampling/importance_sampling_ratio/mean": 0.9929749369621277, "sampling/importance_sampling_ratio/min": 0.4004974663257599, "sampling/sampling_logp_difference/max": 0.9150478839874268, "sampling/sampling_logp_difference/mean": 0.015957921743392944, "step": 111, "step_time": 1.3521366689819843 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 16.0, "completions/max_terminated_length": 16.0, "completions/mean_length": 9.59999942779541, "completions/mean_terminated_length": 9.59999942779541, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.08041965961456299, "epoch": 0.30434782608695654, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4743595.0, "reward": 0.1953125, "reward_std": 0.40248265862464905, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.004687500186264515, "rewards/length_penalty/std": 0.0015783633571118116, "sampling/importance_sampling_ratio/max": 1.2724101543426514, "sampling/importance_sampling_ratio/mean": 0.979505181312561, "sampling/importance_sampling_ratio/min": 0.30284497141838074, "sampling/sampling_logp_difference/max": 1.1945343017578125, "sampling/sampling_logp_difference/mean": 0.03486202657222748, "step": 112, "step_time": 1.4244879351463169 }, { "clip_ratio/high_max": 0.002247191034257412, "clip_ratio/high_mean": 0.002247191034257412, "clip_ratio/low_mean": 0.0024222968146204948, "clip_ratio/low_min": 0.0024222968146204948, "clip_ratio/region_mean": 0.004669487755745649, "completions/clipped_ratio": 0.0, "completions/max_length": 81.0, "completions/max_terminated_length": 81.0, "completions/mean_length": 11.719999313354492, "completions/mean_terminated_length": 11.719999313354492, "completions/min_length": 8.0, "completions/min_terminated_length": 8.0, "entropy": 0.2182243548333645, "epoch": 0.3070652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.03254767879843712, "learning_rate": 5e-05, "loss": 0.004908743780106306, "num_tokens": 4747491.0, "reward": -0.005722655914723873, "reward_std": 0.0064144073985517025, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0057226563803851604, "rewards/length_penalty/std": 0.0064144073985517025, "sampling/importance_sampling_ratio/max": 2.493706464767456, "sampling/importance_sampling_ratio/mean": 0.9865428805351257, "sampling/importance_sampling_ratio/min": 0.045952051877975464, "sampling/sampling_logp_difference/max": 3.0801568031311035, "sampling/sampling_logp_difference/mean": 0.05273308977484703, "step": 113, "step_time": 1.8032670808024704 }, { "clip_ratio/high_max": 0.0025641025975346567, "clip_ratio/high_mean": 0.0025641025975346567, "clip_ratio/low_mean": 0.03051282111555338, "clip_ratio/low_min": 0.03051282111555338, "clip_ratio/region_mean": 0.03307692389935255, "completions/clipped_ratio": 0.0, "completions/max_length": 8.0, "completions/max_terminated_length": 8.0, "completions/mean_length": 7.87999963760376, "completions/mean_terminated_length": 7.87999963760376, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.06522535011172295, "epoch": 0.30978260869565216, "frac_reward_zero_std": 0.0, "grad_norm": 0.018745606765151024, "learning_rate": 5e-05, "loss": -4.46584090241231e-05, "num_tokens": 4750425.0, "reward": -0.003847656073048711, "reward_std": 0.0001602835691301152, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0038476563058793545, "rewards/length_penalty/std": 0.00016028355457819998, "sampling/importance_sampling_ratio/max": 2.473493814468384, "sampling/importance_sampling_ratio/mean": 0.9672911763191223, "sampling/importance_sampling_ratio/min": 0.03405660763382912, "sampling/sampling_logp_difference/max": 3.3797311782836914, "sampling/sampling_logp_difference/mean": 0.13333092629909515, "step": 114, "step_time": 1.3487779938150197 }, { "clip_ratio/high_max": 0.0053693694993853566, "clip_ratio/high_mean": 0.0053693694993853566, "clip_ratio/low_mean": 0.0268108107149601, "clip_ratio/low_min": 0.0268108107149601, "clip_ratio/region_mean": 0.03218018058687448, "completions/clipped_ratio": 0.0, "completions/max_length": 9.0, "completions/max_terminated_length": 9.0, "completions/mean_length": 7.440000057220459, "completions/mean_terminated_length": 7.440000057220459, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.12924398332834244, "epoch": 0.3125, "frac_reward_zero_std": 0.0, "grad_norm": 0.024923890829086304, "learning_rate": 5e-05, "loss": -2.4935703549999744e-05, "num_tokens": 4753027.0, "reward": -0.0036328125279396772, "reward_std": 0.0002639632730279118, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0036328125279396772, "rewards/length_penalty/std": 0.0002639632730279118, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9673813581466675, "sampling/importance_sampling_ratio/min": 0.013750084675848484, "sampling/sampling_logp_difference/max": 4.286710262298584, "sampling/sampling_logp_difference/mean": 0.14096108078956604, "step": 115, "step_time": 1.3418032780755311 }, { "clip_ratio/high_max": 0.02508999891579151, "clip_ratio/high_mean": 0.02508999891579151, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.02508999891579151, "completions/clipped_ratio": 0.0, "completions/max_length": 9.0, "completions/max_terminated_length": 9.0, "completions/mean_length": 7.139999866485596, "completions/mean_terminated_length": 7.139999866485596, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.1305022731423378, "epoch": 0.31521739130434784, "frac_reward_zero_std": 0.0, "grad_norm": 0.019857604056596756, "learning_rate": 5e-05, "loss": 0.00022584707767236978, "num_tokens": 4756274.0, "reward": -0.0034863280598074198, "reward_std": 0.0002791921724565327, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0034863280598074198, "rewards/length_penalty/std": 0.0002791922015603632, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9779207110404968, "sampling/importance_sampling_ratio/min": 0.09788893908262253, "sampling/sampling_logp_difference/max": 2.3239216804504395, "sampling/sampling_logp_difference/mean": 0.10368214547634125, "step": 116, "step_time": 1.3507270456757396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.01988039370626211, "clip_ratio/low_min": 0.01988039370626211, "clip_ratio/region_mean": 0.01988039370626211, "completions/clipped_ratio": 0.0, "completions/max_length": 8.0, "completions/max_terminated_length": 8.0, "completions/mean_length": 7.059999942779541, "completions/mean_terminated_length": 7.059999942779541, "completions/min_length": 7.0, "completions/min_terminated_length": 7.0, "entropy": 0.14805896878242492, "epoch": 0.3179347826086957, "frac_reward_zero_std": 0.0, "grad_norm": 0.02580389380455017, "learning_rate": 5e-05, "loss": 0.00016078897169791162, "num_tokens": 4758807.0, "reward": 0.19655272364616394, "reward_std": 0.4040018618106842, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.0034472655970603228, "rewards/length_penalty/std": 0.00011713767162291333, "sampling/importance_sampling_ratio/max": 2.068840503692627, "sampling/importance_sampling_ratio/mean": 1.007895827293396, "sampling/importance_sampling_ratio/min": 0.5818674564361572, "sampling/sampling_logp_difference/max": 0.7269883155822754, "sampling/sampling_logp_difference/mean": 0.039887771010398865, "step": 117, "step_time": 1.3575295091141015 }, { "clip_ratio/high_max": 0.03157726135104895, "clip_ratio/high_mean": 0.03157726135104895, "clip_ratio/low_mean": 0.002777777798473835, "clip_ratio/low_min": 0.002777777798473835, "clip_ratio/region_mean": 0.03435503896325827, "completions/clipped_ratio": 0.0, "completions/max_length": 11.0, "completions/max_terminated_length": 11.0, "completions/mean_length": 7.019999980926514, "completions/mean_terminated_length": 7.019999980926514, "completions/min_length": 6.0, "completions/min_terminated_length": 6.0, "entropy": 0.2890350580215454, "epoch": 0.32065217391304346, "frac_reward_zero_std": 0.0, "grad_norm": 0.03372028470039368, "learning_rate": 5e-05, "loss": 0.0004409622051753104, "num_tokens": 4762838.0, "reward": -0.0034277343656867743, "reward_std": 0.0004005874216090888, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0034277343656867743, "rewards/length_penalty/std": 0.0004005874216090888, "sampling/importance_sampling_ratio/max": 2.352555751800537, "sampling/importance_sampling_ratio/mean": 0.984298825263977, "sampling/importance_sampling_ratio/min": 0.31722861528396606, "sampling/sampling_logp_difference/max": 1.1481326818466187, "sampling/sampling_logp_difference/mean": 0.09629469364881516, "step": 118, "step_time": 1.4079574490897357 }, { "clip_ratio/high_max": 0.006035719002829865, "clip_ratio/high_mean": 0.006035719002829865, "clip_ratio/low_mean": 0.00014327910321298987, "clip_ratio/low_min": 0.00014327910321298987, "clip_ratio/region_mean": 0.006178998108953238, "completions/clipped_ratio": 0.09999999403953552, "completions/max_length": 2048.0, "completions/max_terminated_length": 66.0, "completions/mean_length": 214.1999969482422, "completions/mean_terminated_length": 10.44444465637207, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.11324097216129303, "epoch": 0.3233695652173913, "frac_reward_zero_std": 0.0, "grad_norm": 0.03372209891676903, "learning_rate": 5e-05, "loss": 0.16911020874977112, "num_tokens": 4777058.0, "reward": -0.08458983898162842, "reward_std": 0.3374756872653961, "rewards/correctness/mean": 0.019999999552965164, "rewards/correctness/std": 0.1414213478565216, "rewards/length_penalty/mean": -0.10458984225988388, "rewards/length_penalty/std": 0.3015563488006592, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9981654286384583, "sampling/importance_sampling_ratio/min": 0.06164000555872917, "sampling/sampling_logp_difference/max": 2.7864441871643066, "sampling/sampling_logp_difference/mean": 0.006951784715056419, "step": 119, "step_time": 21.10556685179472 }, { "clip_ratio/high_max": 0.00357142873108387, "clip_ratio/high_mean": 0.00357142873108387, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.00357142873108387, "completions/clipped_ratio": 0.0, "completions/max_length": 7.0, "completions/max_terminated_length": 7.0, "completions/mean_length": 5.779999732971191, "completions/mean_terminated_length": 5.779999732971191, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.134778194129467, "epoch": 0.32608695652173914, "frac_reward_zero_std": 0.0, "grad_norm": 0.04130018502473831, "learning_rate": 5e-05, "loss": 0.000489944766741246, "num_tokens": 4780197.0, "reward": -0.0028222654946148396, "reward_std": 0.0004760709998663515, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.002822265727445483, "rewards/length_penalty/std": 0.00047607102897018194, "sampling/importance_sampling_ratio/max": 1.3674979209899902, "sampling/importance_sampling_ratio/mean": 0.9822409152984619, "sampling/importance_sampling_ratio/min": 0.2521163523197174, "sampling/sampling_logp_difference/max": 1.3778645992279053, "sampling/sampling_logp_difference/mean": 0.06130911037325859, "step": 120, "step_time": 1.3341772560961545 }, { "clip_ratio/high_max": 0.003333333507180214, "clip_ratio/high_mean": 0.003333333507180214, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.003333333507180214, "completions/clipped_ratio": 0.0, "completions/max_length": 11.0, "completions/max_terminated_length": 11.0, "completions/mean_length": 5.799999713897705, "completions/mean_terminated_length": 5.799999713897705, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.05531938299536705, "epoch": 0.328804347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.018474295735359192, "learning_rate": 5e-05, "loss": 7.076066685840487e-05, "num_tokens": 4783827.0, "reward": -0.002832031110301614, "reward_std": 0.0008134698728099465, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0028320313431322575, "rewards/length_penalty/std": 0.0008134698728099465, "sampling/importance_sampling_ratio/max": 2.6519734859466553, "sampling/importance_sampling_ratio/mean": 0.9936264753341675, "sampling/importance_sampling_ratio/min": 0.16647198796272278, "sampling/sampling_logp_difference/max": 1.7929282188415527, "sampling/sampling_logp_difference/mean": 0.02191774547100067, "step": 121, "step_time": 1.8551505350042135 }, { "clip_ratio/high_max": 0.007999999821186066, "clip_ratio/high_mean": 0.007999999821186066, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.007999999821186066, "completions/clipped_ratio": 0.0, "completions/max_length": 7.0, "completions/max_terminated_length": 7.0, "completions/mean_length": 5.039999961853027, "completions/mean_terminated_length": 5.039999961853027, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.058468326926231384, "epoch": 0.33152173913043476, "frac_reward_zero_std": 0.0, "grad_norm": 0.08369459956884384, "learning_rate": 5e-05, "loss": 0.00031552743166685104, "num_tokens": 4787199.0, "reward": -0.0024609374813735485, "reward_std": 0.00013810678501613438, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0024609374813735485, "rewards/length_penalty/std": 0.00013810678501613438, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9896508455276489, "sampling/importance_sampling_ratio/min": 0.4757717549800873, "sampling/sampling_logp_difference/max": 1.4063184261322021, "sampling/sampling_logp_difference/mean": 0.028557252138853073, "step": 122, "step_time": 1.33938106498681 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.00357142873108387, "clip_ratio/low_min": 0.00357142873108387, "clip_ratio/region_mean": 0.00357142873108387, "completions/clipped_ratio": 0.0, "completions/max_length": 9.0, "completions/max_terminated_length": 9.0, "completions/mean_length": 5.480000019073486, "completions/mean_terminated_length": 5.480000019073486, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.047182951122522354, "epoch": 0.3342391304347826, "frac_reward_zero_std": 0.0, "grad_norm": 0.04056137055158615, "learning_rate": 5e-05, "loss": 0.0001512781309429556, "num_tokens": 4790713.0, "reward": -0.0026757812593132257, "reward_std": 0.0005053228815086186, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0026757812593132257, "rewards/length_penalty/std": 0.0005053228233009577, "sampling/importance_sampling_ratio/max": 3.0, "sampling/importance_sampling_ratio/mean": 0.9900610446929932, "sampling/importance_sampling_ratio/min": 0.1260024756193161, "sampling/sampling_logp_difference/max": 2.071453809738159, "sampling/sampling_logp_difference/mean": 0.061743102967739105, "step": 123, "step_time": 1.3534303947817534 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0037037037312984467, "clip_ratio/low_min": 0.0037037037312984467, "clip_ratio/region_mean": 0.0037037037312984467, "completions/clipped_ratio": 0.0, "completions/max_length": 9.0, "completions/max_terminated_length": 9.0, "completions/mean_length": 5.239999771118164, "completions/mean_terminated_length": 5.239999771118164, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.035425788909196856, "epoch": 0.33695652173913043, "frac_reward_zero_std": 0.0, "grad_norm": 0.019155023619532585, "learning_rate": 5e-05, "loss": 0.0002729708794504404, "num_tokens": 4793225.0, "reward": -0.002558593638241291, "reward_std": 0.0004249873454682529, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.002558593638241291, "rewards/length_penalty/std": 0.0004249873454682529, "sampling/importance_sampling_ratio/max": 1.146919846534729, "sampling/importance_sampling_ratio/mean": 0.9921718835830688, "sampling/importance_sampling_ratio/min": 0.2001161277294159, "sampling/sampling_logp_difference/max": 1.6088573932647705, "sampling/sampling_logp_difference/mean": 0.021887296810746193, "step": 124, "step_time": 1.3246259889565408 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.0003937849251087755, "epoch": 0.33967391304347827, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4795955.0, "reward": 0.19755859673023224, "reward_std": 0.40406104922294617, "rewards/correctness/mean": 0.20000000298023224, "rewards/correctness/std": 0.4040610194206238, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999623894691467, "sampling/importance_sampling_ratio/min": 0.9991763830184937, "sampling/sampling_logp_difference/max": 0.000823974609375, "sampling/sampling_logp_difference/mean": 3.765869405469857e-05, "step": 125, "step_time": 1.3312179518397897 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.019999999552965164, "completions/max_length": 2048.0, "completions/max_terminated_length": 31.0, "completions/mean_length": 46.37999725341797, "completions/mean_terminated_length": 5.530611991882324, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.070949225127697, "epoch": 0.3423913043478261, "frac_reward_zero_std": 0.0, "grad_norm": 0.021165043115615845, "learning_rate": 5e-05, "loss": 0.13177801668643951, "num_tokens": 4803244.0, "reward": -0.022646483033895493, "reward_std": 0.14105087518692017, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.022646484896540642, "rewards/length_penalty/std": 0.14105086028575897, "sampling/importance_sampling_ratio/max": 1.466500163078308, "sampling/importance_sampling_ratio/mean": 0.996273934841156, "sampling/importance_sampling_ratio/min": 0.1469108611345291, "sampling/sampling_logp_difference/max": 1.9179292917251587, "sampling/sampling_logp_difference/mean": 0.0077306954190135, "step": 126, "step_time": 19.41571421804838 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.00010384343331679702, "epoch": 0.3451086956521739, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4806124.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999924302101135, "sampling/importance_sampling_ratio/min": 0.9999542832374573, "sampling/sampling_logp_difference/max": 4.57763671875e-05, "sampling/sampling_logp_difference/mean": 7.62939453125e-06, "step": 127, "step_time": 1.3068981308024377 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.0008381461084354669, "epoch": 0.34782608695652173, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4808694.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999207854270935, "sampling/importance_sampling_ratio/min": 0.9987267255783081, "sampling/sampling_logp_difference/max": 0.00127410888671875, "sampling/sampling_logp_difference/mean": 7.928466948214918e-05, "step": 128, "step_time": 1.3141667230520397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.003685102250892669, "epoch": 0.35054347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4812004.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9993257522583008, "sampling/importance_sampling_ratio/min": 0.9812495112419128, "sampling/sampling_logp_difference/max": 0.01892852783203125, "sampling/sampling_logp_difference/mean": 0.0006792221101932228, "step": 129, "step_time": 1.3033307418227196 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.005226453929208219, "epoch": 0.3532608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4815024.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9988739490509033, "sampling/importance_sampling_ratio/min": 0.9655466675758362, "sampling/sampling_logp_difference/max": 0.035060882568359375, "sampling/sampling_logp_difference/mean": 0.0011407547863200307, "step": 130, "step_time": 1.3168136766180396 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.0034834542311728, "epoch": 0.35597826086956524, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4818654.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9995189905166626, "sampling/importance_sampling_ratio/min": 0.9917525053024292, "sampling/sampling_logp_difference/max": 0.008281707763671875, "sampling/sampling_logp_difference/mean": 0.0004819030873477459, "step": 131, "step_time": 1.2911464849021286 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.014990408346056939, "epoch": 0.358695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4820734.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9970442652702332, "sampling/importance_sampling_ratio/min": 0.9578938484191895, "sampling/sampling_logp_difference/max": 0.043018341064453125, "sampling/sampling_logp_difference/mean": 0.0029887238051742315, "step": 132, "step_time": 1.30298503045924 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 5.0, "completions/mean_terminated_length": 5.0, "completions/min_length": 5.0, "completions/min_terminated_length": 5.0, "entropy": 0.02549996804445982, "epoch": 0.36141304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4823674.0, "reward": -0.00244140625, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00244140625, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9941732287406921, "sampling/importance_sampling_ratio/min": 0.8892385959625244, "sampling/sampling_logp_difference/max": 0.11738967895507812, "sampling/sampling_logp_difference/mean": 0.005998215172439814, "step": 133, "step_time": 1.7252554318401963 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.004347826167941093, "clip_ratio/low_min": 0.004347826167941093, "clip_ratio/region_mean": 0.004347826167941093, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 4.440000057220459, "completions/mean_terminated_length": 4.440000057220459, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.07134933695197106, "epoch": 0.3641304347826087, "frac_reward_zero_std": 0.0, "grad_norm": 0.04109515994787216, "learning_rate": 5e-05, "loss": 0.00047914363676682115, "num_tokens": 4826586.0, "reward": -0.0021679687779396772, "reward_std": 0.0006845894968137145, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0021679687779396772, "rewards/length_penalty/std": 0.0006845894968137145, "sampling/importance_sampling_ratio/max": 1.6961158514022827, "sampling/importance_sampling_ratio/mean": 0.9964490532875061, "sampling/importance_sampling_ratio/min": 0.6559172868728638, "sampling/sampling_logp_difference/max": 0.5283408164978027, "sampling/sampling_logp_difference/mean": 0.026138095185160637, "step": 134, "step_time": 1.2979106933344156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.013333334028720856, "clip_ratio/low_min": 0.013333334028720856, "clip_ratio/region_mean": 0.013333334028720856, "completions/clipped_ratio": 0.0, "completions/max_length": 5.0, "completions/max_terminated_length": 5.0, "completions/mean_length": 2.119999885559082, "completions/mean_terminated_length": 2.119999885559082, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.1339002564549446, "epoch": 0.36684782608695654, "frac_reward_zero_std": 0.0, "grad_norm": 0.033667221665382385, "learning_rate": 5e-05, "loss": 0.0005218620062805712, "num_tokens": 4830872.0, "reward": -0.0010351561941206455, "reward_std": 0.0008858542423695326, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.0010351561941206455, "rewards/length_penalty/std": 0.0008858542423695326, "sampling/importance_sampling_ratio/max": 2.1150805950164795, "sampling/importance_sampling_ratio/mean": 0.9666422605514526, "sampling/importance_sampling_ratio/min": 0.19639620184898376, "sampling/sampling_logp_difference/max": 1.6276211738586426, "sampling/sampling_logp_difference/mean": 0.18057852983474731, "step": 135, "step_time": 1.364359324099496 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.000582211505388841, "epoch": 0.3695652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4833002.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 0.9999809265136719, "sampling/importance_sampling_ratio/mean": 0.9999510049819946, "sampling/importance_sampling_ratio/min": 0.9999046921730042, "sampling/sampling_logp_difference/max": 9.5367431640625e-05, "sampling/sampling_logp_difference/mean": 4.898071347270161e-05, "step": 136, "step_time": 1.2746044367086142 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.0007837315904907882, "epoch": 0.37228260869565216, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4835912.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 0.9999847412109375, "sampling/importance_sampling_ratio/mean": 0.9999340772628784, "sampling/importance_sampling_ratio/min": 0.9998207688331604, "sampling/sampling_logp_difference/max": 0.000179290771484375, "sampling/sampling_logp_difference/mean": 6.591796409338713e-05, "step": 137, "step_time": 1.2879312569275498 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 8.089616167126224e-05, "epoch": 0.375, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4838992.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999938607215881, "sampling/importance_sampling_ratio/min": 0.9999809265136719, "sampling/sampling_logp_difference/max": 1.9073486328125e-05, "sampling/sampling_logp_difference/mean": 6.103515261202119e-06, "step": 138, "step_time": 1.2414450428914279 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.00010406925866845995, "epoch": 0.37771739130434784, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4841822.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999917149543762, "sampling/importance_sampling_ratio/min": 0.9999809265136719, "sampling/sampling_logp_difference/max": 1.9073486328125e-05, "sampling/sampling_logp_difference/mean": 8.239745511673391e-06, "step": 139, "step_time": 1.2661544231232256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 4.848098687944002e-05, "epoch": 0.3804347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4845512.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999963641166687, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 3.585815420592553e-06, "step": 140, "step_time": 1.2740853056311607 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.83553352043964e-05, "epoch": 0.38315217391304346, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4848292.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999962449073792, "sampling/importance_sampling_ratio/min": 0.9999885559082031, "sampling/sampling_logp_difference/max": 1.1444091796875e-05, "sampling/sampling_logp_difference/mean": 3.7384031656984007e-06, "step": 141, "step_time": 1.2378920100163668 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 4.5442912596627136e-05, "epoch": 0.3858695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4851132.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999971389770508, "sampling/importance_sampling_ratio/min": 0.9999847412109375, "sampling/sampling_logp_difference/max": 1.52587890625e-05, "sampling/sampling_logp_difference/mean": 2.822876012942288e-06, "step": 142, "step_time": 1.2508271427359432 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.5392274437763263e-05, "epoch": 0.38858695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4853522.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 9.155273232863692e-07, "step": 143, "step_time": 1.2680298667401075 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.9370828522369265e-05, "epoch": 0.391304347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4856622.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999991655349731, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 8.392333938900265e-07, "step": 144, "step_time": 1.2653971719555557 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.0671510719694196e-05, "epoch": 0.39402173913043476, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4859002.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999986886978149, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 1.2969970839549205e-06, "step": 145, "step_time": 1.6912914318963885 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 9.670421786722726e-06, "epoch": 0.3967391304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4861992.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 146, "step_time": 1.3013200680725276 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.4405572185060008e-05, "epoch": 0.39945652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4864342.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999985098838806, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 1.449584942747606e-06, "step": 147, "step_time": 1.2580423734616488 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.3304339154274203e-05, "epoch": 0.40217391304347827, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4867302.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 148, "step_time": 1.2458707271143794 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.74797240915359e-06, "epoch": 0.4048913043478261, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4870322.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 149, "step_time": 1.2536773330066353 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.4288289457908832e-05, "epoch": 0.4076086956521739, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4872132.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 150, "step_time": 1.253505111206323 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.1226110655115917e-05, "epoch": 0.41032608695652173, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4875152.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999995231628418, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 4.577636616431846e-07, "step": 151, "step_time": 1.2520484649576247 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.643265495658852e-05, "epoch": 0.41304347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4878382.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.999999463558197, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 5.340576194612368e-07, "step": 152, "step_time": 1.2810317238327116 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 6.8622498474724125e-06, "epoch": 0.4157608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4881952.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 153, "step_time": 1.2494909665547311 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 7.287962307600537e-06, "epoch": 0.41847826086956524, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4885762.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 154, "step_time": 1.2711481822188944 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 3.968705859733745e-05, "epoch": 0.421195652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4888842.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999975562095642, "sampling/importance_sampling_ratio/min": 0.9999847412109375, "sampling/sampling_logp_difference/max": 1.52587890625e-05, "sampling/sampling_logp_difference/mean": 2.441406195430318e-06, "step": 155, "step_time": 1.2514137201942503 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.3754256542597432e-05, "epoch": 0.42391304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4892452.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 156, "step_time": 1.6910824335645884 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 3.7028261795057914e-05, "epoch": 0.4266304347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4895182.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999976754188538, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 2.2888182229507947e-06, "step": 157, "step_time": 1.2348916861228645 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.0749504872364923e-05, "epoch": 0.42934782608695654, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4898642.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999992251396179, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 7.629394076502649e-07, "step": 158, "step_time": 1.2514287990052253 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 7.847922643122728e-06, "epoch": 0.4320652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4901962.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 159, "step_time": 1.267498184228316 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.326207220699871e-05, "epoch": 0.43478260869565216, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4905562.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999995231628418, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 4.577636616431846e-07, "step": 160, "step_time": 1.2317475187592208 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.0631144323269837e-05, "epoch": 0.4375, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4909042.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999995231628418, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 4.577636616431846e-07, "step": 161, "step_time": 1.2522022440098226 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.977814871584997e-05, "epoch": 0.44021739130434784, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4911682.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999954700469971, "sampling/importance_sampling_ratio/min": 0.9999847412109375, "sampling/sampling_logp_difference/max": 1.52587890625e-05, "sampling/sampling_logp_difference/mean": 4.501342573348666e-06, "step": 162, "step_time": 1.262192539870739 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.351954724668758e-05, "epoch": 0.4429347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4914442.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999997019767761, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 3.0517577442878974e-07, "step": 163, "step_time": 1.270949344150722 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 4.7593245835741985e-05, "epoch": 0.44565217391304346, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4917352.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999963045120239, "sampling/importance_sampling_ratio/min": 0.9999885559082031, "sampling/sampling_logp_difference/max": 1.1444091796875e-05, "sampling/sampling_logp_difference/mean": 3.662109293145477e-06, "step": 164, "step_time": 1.2493651057593524 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.129057356796693e-05, "epoch": 0.4483695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4920772.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999986290931702, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 1.3732909565078444e-06, "step": 165, "step_time": 1.2717818422243 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.9856214385072236e-05, "epoch": 0.45108695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4922732.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999985098838806, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 1.449584942747606e-06, "step": 166, "step_time": 1.2344487588852644 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 6.408186745829881e-05, "epoch": 0.453804347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4926202.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999957084655762, "sampling/importance_sampling_ratio/min": 0.9999847412109375, "sampling/sampling_logp_difference/max": 1.52587890625e-05, "sampling/sampling_logp_difference/mean": 4.272460955689894e-06, "step": 167, "step_time": 1.253273184178397 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 3.254499351896811e-05, "epoch": 0.45652173913043476, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4929092.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.999998152256012, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 1.8310546465727384e-06, "step": 168, "step_time": 1.6192582400981337 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.3008329187869094e-05, "epoch": 0.4592391304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4932212.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 169, "step_time": 1.2479304489679635 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.0525663128646556e-05, "epoch": 0.46195652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4934512.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 170, "step_time": 1.233439117204398 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 4.018152976641432e-05, "epoch": 0.46467391304347827, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4937332.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999973177909851, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 2.670288040462765e-06, "step": 171, "step_time": 1.248290613759309 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 9.258378850063309e-05, "epoch": 0.4673913043478261, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4940882.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999923706054688, "sampling/importance_sampling_ratio/min": 0.999969482421875, "sampling/sampling_logp_difference/max": 3.0517578125e-05, "sampling/sampling_logp_difference/mean": 7.62939453125e-06, "step": 172, "step_time": 1.27427533804439 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 9.897275594994425e-06, "epoch": 0.4701086956521739, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4944222.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 173, "step_time": 1.2721459951717407 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.1621312114584726e-05, "epoch": 0.47282608695652173, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4946512.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 174, "step_time": 1.2442881846800447 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.7290810248814523e-05, "epoch": 0.47554347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4949642.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 9.155273232863692e-07, "step": 175, "step_time": 1.2409526768606156 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 9.690917795524002e-06, "epoch": 0.4782608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4952332.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 176, "step_time": 1.261909464839846 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.2395698104228358e-05, "epoch": 0.48097826086956524, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4954612.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 177, "step_time": 1.246021990897134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.326562895788811e-05, "epoch": 0.483695652173913, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4957122.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999968409538269, "sampling/importance_sampling_ratio/min": 0.9999809265136719, "sampling/sampling_logp_difference/max": 1.9073486328125e-05, "sampling/sampling_logp_difference/mean": 3.128051730527659e-06, "step": 178, "step_time": 1.2583061011973768 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.6112725097627846e-05, "epoch": 0.48641304347826086, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4959972.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999992847442627, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 6.866454782539222e-07, "step": 179, "step_time": 1.2606565097812563 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.8233689560438504e-05, "epoch": 0.4891304347826087, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4964482.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999958872795105, "sampling/importance_sampling_ratio/min": 0.999977171421051, "sampling/sampling_logp_difference/max": 2.288818359375e-05, "sampling/sampling_logp_difference/mean": 4.1198727558366954e-06, "step": 180, "step_time": 1.7697891714051366 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.0001358017566417402, "epoch": 0.49184782608695654, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4967692.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999897480010986, "sampling/importance_sampling_ratio/min": 0.9999351501464844, "sampling/sampling_logp_difference/max": 6.4849853515625e-05, "sampling/sampling_logp_difference/mean": 1.0223388017038815e-05, "step": 181, "step_time": 1.281004831660539 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.3414277166011743e-05, "epoch": 0.4945652173913043, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4970582.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999987483024597, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 1.220703097715159e-06, "step": 182, "step_time": 1.2411391898058355 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.549687185615767e-05, "epoch": 0.49728260869565216, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4972922.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999992251396179, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 7.629394076502649e-07, "step": 183, "step_time": 1.2466163753997535 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.2948591140448116e-05, "epoch": 0.5, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4975722.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999985098838806, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 1.449584942747606e-06, "step": 184, "step_time": 1.2495241009164602 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 0.00011228810617467389, "epoch": 0.5027173913043478, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4979492.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999907612800598, "sampling/importance_sampling_ratio/min": 0.9999542832374573, "sampling/sampling_logp_difference/max": 4.57763671875e-05, "sampling/sampling_logp_difference/mean": 9.231566764356103e-06, "step": 185, "step_time": 1.2547712451778352 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.8512038231128825e-05, "epoch": 0.5054347826086957, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4983612.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999963045120239, "sampling/importance_sampling_ratio/min": 0.9999809265136719, "sampling/sampling_logp_difference/max": 1.9073486328125e-05, "sampling/sampling_logp_difference/mean": 3.662109293145477e-06, "step": 186, "step_time": 1.2657833849079907 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.952836181677412e-06, "epoch": 0.5081521739130435, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4986722.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 187, "step_time": 1.244236164726317 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.9592240207421127e-05, "epoch": 0.5108695652173914, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4990692.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999979138374329, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 2.0599363779183477e-06, "step": 188, "step_time": 1.319129457231611 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 7.066765920171747e-06, "epoch": 0.5135869565217391, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4993842.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 189, "step_time": 1.2519651120528579 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 8.771297689236236e-06, "epoch": 0.5163043478260869, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4996452.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 190, "step_time": 1.273618385894224 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 4.539682049653493e-05, "epoch": 0.5190217391304348, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 4998812.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.999997079372406, "sampling/importance_sampling_ratio/min": 0.9999847412109375, "sampling/sampling_logp_difference/max": 1.52587890625e-05, "sampling/sampling_logp_difference/mean": 2.899169885495212e-06, "step": 191, "step_time": 1.249575928086415 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.1945640471822116e-05, "epoch": 0.5217391304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5003662.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.999999463558197, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 5.340576194612368e-07, "step": 192, "step_time": 1.8078890063334256 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.3028971554595046e-05, "epoch": 0.5244565217391305, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5005842.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999991655349731, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 8.392333938900265e-07, "step": 193, "step_time": 1.2710501928813756 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 2.1131416997377527e-05, "epoch": 0.5271739130434783, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5009552.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999990463256836, "sampling/importance_sampling_ratio/min": 0.9999923706054688, "sampling/sampling_logp_difference/max": 7.62939453125e-06, "sampling/sampling_logp_difference/mean": 9.155273232863692e-07, "step": 194, "step_time": 1.2856320531573147 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 5.9920400781265924e-05, "epoch": 0.529891304347826, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5011962.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.999994158744812, "sampling/importance_sampling_ratio/min": 0.9999656677246094, "sampling/sampling_logp_difference/max": 3.4332275390625e-05, "sampling/sampling_logp_difference/mean": 5.798339770990424e-06, "step": 195, "step_time": 1.2224214975722134 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.3486030820786254e-05, "epoch": 0.532608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5014482.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999995231628418, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 4.577636616431846e-07, "step": 196, "step_time": 1.2343761902302504 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 9.439519999432378e-06, "epoch": 0.5353260869565217, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5017932.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 197, "step_time": 1.251520578050986 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.2180034536868333e-05, "epoch": 0.5380434782608695, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5023892.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.999999463558197, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 5.340576194612368e-07, "step": 198, "step_time": 1.4176248768344522 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 1.558009098516777e-05, "epoch": 0.5407608695652174, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5026632.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 0.9999988079071045, "sampling/importance_sampling_ratio/min": 0.9999962449073792, "sampling/sampling_logp_difference/max": 3.814697265625e-06, "sampling/sampling_logp_difference/mean": 1.1444091114753974e-06, "step": 199, "step_time": 1.2287625020835549 }, { "clip_ratio/high_max": 0.0, "clip_ratio/high_mean": 0.0, "clip_ratio/low_mean": 0.0, "clip_ratio/low_min": 0.0, "clip_ratio/region_mean": 0.0, "completions/clipped_ratio": 0.0, "completions/max_length": 1.0, "completions/max_terminated_length": 1.0, "completions/mean_length": 1.0, "completions/mean_terminated_length": 1.0, "completions/min_length": 1.0, "completions/min_terminated_length": 1.0, "entropy": 9.348761159344576e-06, "epoch": 0.5434782608695652, "frac_reward_zero_std": 1.0, "grad_norm": 0.0, "learning_rate": 5e-05, "loss": 0.0, "num_tokens": 5030942.0, "reward": -0.00048828125, "reward_std": 0.0, "rewards/correctness/mean": 0.0, "rewards/correctness/std": 0.0, "rewards/length_penalty/mean": -0.00048828125, "rewards/length_penalty/std": 0.0, "sampling/importance_sampling_ratio/max": 1.0, "sampling/importance_sampling_ratio/mean": 1.0, "sampling/importance_sampling_ratio/min": 1.0, "sampling/sampling_logp_difference/max": 0.0, "sampling/sampling_logp_difference/mean": 0.0, "step": 200, "step_time": 1.3342331610620022 } ], "logging_steps": 1, "max_steps": 200, "num_input_tokens_seen": 5030942, "num_train_epochs": 1, "save_steps": 50, "stateful_callbacks": { "TrainerControl": { "args": { "should_epoch_stop": false, "should_evaluate": false, "should_log": false, "should_save": true, "should_training_stop": true }, "attributes": {} } }, "total_flos": 0.0, "train_batch_size": 10, "trial_name": null, "trial_params": null }