| { |
| "best_global_step": null, |
| "best_metric": null, |
| "best_model_checkpoint": null, |
| "epoch": 0.0005, |
| "eval_steps": 500, |
| "global_step": 50, |
| "is_hyper_param_search": false, |
| "is_local_process_zero": true, |
| "is_world_process_zero": true, |
| "log_history": [ |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9365.0, |
| "completions/max_terminated_length": 9365.0, |
| "completions/mean_length": 8020.65625, |
| "completions/mean_terminated_length": 8020.65625, |
| "completions/min_length": 4249.0, |
| "completions/min_terminated_length": 4249.0, |
| "entropy": 0.09263912006281316, |
| "epoch": 1e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.3079737424850464, |
| "kl": 0.0, |
| "learning_rate": 0.0, |
| "loss": -0.0363, |
| "num_tokens": 283668.0, |
| "reward": -0.4597676992416382, |
| "reward_std": 0.3370218873023987, |
| "rewards/rollout_reward_func/mean": -0.4597676992416382, |
| "rewards/rollout_reward_func/std": 0.37389153242111206, |
| "sampling/importance_sampling_ratio/max": 1.752305507659912, |
| "sampling/importance_sampling_ratio/mean": 0.9996651411056519, |
| "sampling/importance_sampling_ratio/min": 0.4149494767189026, |
| "sampling/sampling_logp_difference/max": 0.8795984983444214, |
| "sampling/sampling_logp_difference/mean": 0.009596828371286392, |
| "step": 1, |
| "step_time": 96.616219112002 |
| }, |
| { |
| "clip_ratio/high_max": 0.0, |
| "clip_ratio/high_mean": 0.0, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0, |
| "entropy": 0.09263912006281316, |
| "epoch": 2e-05, |
| "grad_norm": 1.297884225845337, |
| "kl": 0.0, |
| "learning_rate": 2.8571428571428573e-06, |
| "loss": -0.0363, |
| "step": 2, |
| "step_time": 28.28962996199698 |
| }, |
| { |
| "clip_ratio/high_max": 0.005740102380514145, |
| "clip_ratio/high_mean": 0.0028700511902570724, |
| "clip_ratio/low_mean": 0.0018344554118812084, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004704506660345942, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9371.0, |
| "completions/max_terminated_length": 9371.0, |
| "completions/mean_length": 7904.6875, |
| "completions/mean_terminated_length": 7904.6875, |
| "completions/min_length": 2043.0, |
| "completions/min_terminated_length": 2043.0, |
| "entropy": 0.11998538533225656, |
| "epoch": 3e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.5289855003356934, |
| "kl": 0.0012301416445552604, |
| "learning_rate": 5.7142857142857145e-06, |
| "loss": -0.0457, |
| "num_tokens": 564087.0, |
| "reward": -0.5441880226135254, |
| "reward_std": 0.3924558162689209, |
| "rewards/rollout_reward_func/mean": -0.5441880226135254, |
| "rewards/rollout_reward_func/std": 0.42554038763046265, |
| "sampling/importance_sampling_ratio/max": 1.5868604183197021, |
| "sampling/importance_sampling_ratio/mean": 0.9994746446609497, |
| "sampling/importance_sampling_ratio/min": 0.5226752758026123, |
| "sampling/sampling_logp_difference/max": 0.6487948894500732, |
| "sampling/sampling_logp_difference/mean": 0.011393455788493156, |
| "step": 3, |
| "step_time": 96.45885010200436 |
| }, |
| { |
| "clip_ratio/high_max": 0.005765739129856229, |
| "clip_ratio/high_mean": 0.0028828695649281144, |
| "clip_ratio/low_mean": 0.0021901361469645053, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00507300574099645, |
| "entropy": 0.12003705860115588, |
| "epoch": 4e-05, |
| "grad_norm": 1.2704079151153564, |
| "kl": 0.001491897305186285, |
| "learning_rate": 8.571428571428573e-06, |
| "loss": -0.047, |
| "step": 4, |
| "step_time": 29.195899179991102 |
| }, |
| { |
| "clip_ratio/high_max": 0.005963329633232206, |
| "clip_ratio/high_mean": 0.002981664816616103, |
| "clip_ratio/low_mean": 0.0014880952658131719, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.004469760082429275, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8882.0, |
| "completions/max_terminated_length": 8882.0, |
| "completions/mean_length": 7734.1875, |
| "completions/mean_terminated_length": 7734.1875, |
| "completions/min_length": 1783.0, |
| "completions/min_terminated_length": 1783.0, |
| "entropy": 0.09044346120208502, |
| "epoch": 5e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.5453312397003174, |
| "kl": 0.0007876164445406175, |
| "learning_rate": 1.1428571428571429e-05, |
| "loss": -0.0298, |
| "num_tokens": 838350.0, |
| "reward": -0.5028736591339111, |
| "reward_std": 0.371159166097641, |
| "rewards/rollout_reward_func/mean": -0.5028736591339111, |
| "rewards/rollout_reward_func/std": 0.38670846819877625, |
| "sampling/importance_sampling_ratio/max": 1.758213996887207, |
| "sampling/importance_sampling_ratio/mean": 0.9994957447052002, |
| "sampling/importance_sampling_ratio/min": 0.3028993308544159, |
| "sampling/sampling_logp_difference/max": 1.194354772567749, |
| "sampling/sampling_logp_difference/mean": 0.009732999838888645, |
| "step": 5, |
| "step_time": 93.71373151599619 |
| }, |
| { |
| "clip_ratio/high_max": 0.006641630490776151, |
| "clip_ratio/high_mean": 0.004027052520541474, |
| "clip_ratio/low_mean": 0.002363648993195966, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00639070151373744, |
| "entropy": 0.08915826864540577, |
| "epoch": 6e-05, |
| "grad_norm": 1.28467857837677, |
| "kl": 0.0010278278514306294, |
| "learning_rate": 1.4285714285714285e-05, |
| "loss": -0.0305, |
| "step": 6, |
| "step_time": 27.6442580659932 |
| }, |
| { |
| "clip_ratio/high_max": 0.006255197920836508, |
| "clip_ratio/high_mean": 0.003127598960418254, |
| "clip_ratio/low_mean": 0.0010739664430730045, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.004201565403491259, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9327.0, |
| "completions/max_terminated_length": 9327.0, |
| "completions/mean_length": 7944.875, |
| "completions/mean_terminated_length": 7944.875, |
| "completions/min_length": 1618.0, |
| "completions/min_terminated_length": 1618.0, |
| "entropy": 0.0936605577589944, |
| "epoch": 7e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.6153287887573242, |
| "kl": 0.001322501270351495, |
| "learning_rate": 1.7142857142857145e-05, |
| "loss": -0.04, |
| "num_tokens": 1120382.0, |
| "reward": -0.40193918347358704, |
| "reward_std": 0.42798829078674316, |
| "rewards/rollout_reward_func/mean": -0.40193918347358704, |
| "rewards/rollout_reward_func/std": 0.45951688289642334, |
| "sampling/importance_sampling_ratio/max": 2.626508951187134, |
| "sampling/importance_sampling_ratio/mean": 0.9997774362564087, |
| "sampling/importance_sampling_ratio/min": 0.4664866328239441, |
| "sampling/sampling_logp_difference/max": 0.9656555652618408, |
| "sampling/sampling_logp_difference/mean": 0.011292756535112858, |
| "step": 7, |
| "step_time": 93.59508053499303 |
| }, |
| { |
| "clip_ratio/high_max": 0.004923702392261475, |
| "clip_ratio/high_mean": 0.0028252233169041574, |
| "clip_ratio/low_mean": 0.006011958757881075, |
| "clip_ratio/low_min": 0.0007102272938936949, |
| "clip_ratio/region_mean": 0.008837182191200554, |
| "entropy": 0.08954936638474464, |
| "epoch": 8e-05, |
| "grad_norm": 1.4120911359786987, |
| "kl": 0.0029542466588736715, |
| "learning_rate": 2e-05, |
| "loss": -0.0404, |
| "step": 8, |
| "step_time": 28.796026483003516 |
| }, |
| { |
| "clip_ratio/high_max": 0.004531709826551378, |
| "clip_ratio/high_mean": 0.00261697854148224, |
| "clip_ratio/low_mean": 0.0036588753282558173, |
| "clip_ratio/low_min": 0.0014044943964108825, |
| "clip_ratio/region_mean": 0.006275853869738057, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9217.0, |
| "completions/max_terminated_length": 9217.0, |
| "completions/mean_length": 7774.4375, |
| "completions/mean_terminated_length": 7774.4375, |
| "completions/min_length": 860.0, |
| "completions/min_terminated_length": 860.0, |
| "entropy": 0.09250054159201682, |
| "epoch": 9e-05, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.8319895267486572, |
| "kl": 0.003183710745361168, |
| "learning_rate": 2.2857142857142858e-05, |
| "loss": -0.0399, |
| "num_tokens": 1396422.0, |
| "reward": -0.35600194334983826, |
| "reward_std": 0.46349450945854187, |
| "rewards/rollout_reward_func/mean": -0.35600194334983826, |
| "rewards/rollout_reward_func/std": 0.4859367907047272, |
| "sampling/importance_sampling_ratio/max": 2.4923911094665527, |
| "sampling/importance_sampling_ratio/mean": 0.9995814561843872, |
| "sampling/importance_sampling_ratio/min": 0.34728506207466125, |
| "sampling/sampling_logp_difference/max": 1.0576093196868896, |
| "sampling/sampling_logp_difference/mean": 0.011536622419953346, |
| "step": 9, |
| "step_time": 91.41392734301189 |
| }, |
| { |
| "clip_ratio/high_max": 0.005798806145321578, |
| "clip_ratio/high_mean": 0.0032545167196076363, |
| "clip_ratio/low_mean": 0.005793917312985286, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.009048434003489092, |
| "entropy": 0.08986990223638713, |
| "epoch": 0.0001, |
| "grad_norm": 1.1754401922225952, |
| "kl": 0.006458986033976544, |
| "learning_rate": 2.5714285714285714e-05, |
| "loss": -0.0398, |
| "step": 10, |
| "step_time": 27.723220527001104 |
| }, |
| { |
| "clip_ratio/high_max": 0.002881916589103639, |
| "clip_ratio/high_mean": 0.0014409582945518196, |
| "clip_ratio/low_mean": 0.0003511235991027206, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0017920818936545402, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9155.0, |
| "completions/max_terminated_length": 9155.0, |
| "completions/mean_length": 7173.3125, |
| "completions/mean_terminated_length": 7173.3125, |
| "completions/min_length": 887.0, |
| "completions/min_terminated_length": 887.0, |
| "entropy": 0.06778168666642159, |
| "epoch": 0.00011, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0174713134765625, |
| "kl": 0.004350664900812262, |
| "learning_rate": 2.857142857142857e-05, |
| "loss": -0.2438, |
| "num_tokens": 1653204.0, |
| "reward": -0.4879741668701172, |
| "reward_std": 0.49708276987075806, |
| "rewards/rollout_reward_func/mean": -0.4879741668701172, |
| "rewards/rollout_reward_func/std": 0.5053675770759583, |
| "sampling/importance_sampling_ratio/max": 1.3829487562179565, |
| "sampling/importance_sampling_ratio/mean": 1.0001708269119263, |
| "sampling/importance_sampling_ratio/min": 0.6129343509674072, |
| "sampling/sampling_logp_difference/max": 0.48949742317199707, |
| "sampling/sampling_logp_difference/mean": 0.006062386557459831, |
| "step": 11, |
| "step_time": 88.94538035600272 |
| }, |
| { |
| "clip_ratio/high_max": 0.0021720757358707488, |
| "clip_ratio/high_mean": 0.0010860378679353744, |
| "clip_ratio/low_mean": 0.005371870560338721, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.006457908428274095, |
| "entropy": 0.0648728254600428, |
| "epoch": 0.00012, |
| "grad_norm": 0.9108887910842896, |
| "kl": 0.010094417615619022, |
| "learning_rate": 3.142857142857143e-05, |
| "loss": -0.2448, |
| "step": 12, |
| "step_time": 27.213586286001373 |
| }, |
| { |
| "clip_ratio/high_max": 0.009980108763556927, |
| "clip_ratio/high_mean": 0.004990054381778464, |
| "clip_ratio/low_mean": 0.00396484971861355, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.008954904100392014, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9369.0, |
| "completions/max_terminated_length": 9369.0, |
| "completions/mean_length": 8193.15625, |
| "completions/mean_terminated_length": 8193.15625, |
| "completions/min_length": 960.0, |
| "completions/min_terminated_length": 960.0, |
| "entropy": 0.10193604091182351, |
| "epoch": 0.00013, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.485856294631958, |
| "kl": 0.030039603414479643, |
| "learning_rate": 3.428571428571429e-05, |
| "loss": -0.1495, |
| "num_tokens": 1943132.0, |
| "reward": -0.5804459452629089, |
| "reward_std": 0.27082452178001404, |
| "rewards/rollout_reward_func/mean": -0.5804459452629089, |
| "rewards/rollout_reward_func/std": 0.2952437400817871, |
| "sampling/importance_sampling_ratio/max": 1.951629877090454, |
| "sampling/importance_sampling_ratio/mean": 0.9995810985565186, |
| "sampling/importance_sampling_ratio/min": 0.08306944370269775, |
| "sampling/sampling_logp_difference/max": 2.4880783557891846, |
| "sampling/sampling_logp_difference/mean": 0.016808483749628067, |
| "step": 13, |
| "step_time": 94.51923282000644 |
| }, |
| { |
| "clip_ratio/high_max": 0.015696270624175668, |
| "clip_ratio/high_mean": 0.007848135312087834, |
| "clip_ratio/low_mean": 0.00827725778799504, |
| "clip_ratio/low_min": 0.0021828330936841667, |
| "clip_ratio/region_mean": 0.016125393158290535, |
| "entropy": 0.10211672727018595, |
| "epoch": 0.00014, |
| "grad_norm": 1.529998779296875, |
| "kl": 0.045463139831554145, |
| "learning_rate": 3.7142857142857143e-05, |
| "loss": -0.1549, |
| "step": 14, |
| "step_time": 27.938481579996733 |
| }, |
| { |
| "clip_ratio/high_max": 0.002873942954465747, |
| "clip_ratio/high_mean": 0.0017880951054394245, |
| "clip_ratio/low_mean": 0.0017879052029456943, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003576000308385119, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9252.0, |
| "completions/max_terminated_length": 9252.0, |
| "completions/mean_length": 7854.34375, |
| "completions/mean_terminated_length": 7854.34375, |
| "completions/min_length": 1084.0, |
| "completions/min_terminated_length": 1084.0, |
| "entropy": 0.064073825487867, |
| "epoch": 0.00015, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 2.1420814990997314, |
| "kl": 0.013782255366095342, |
| "learning_rate": 4e-05, |
| "loss": 0.0498, |
| "num_tokens": 2221867.0, |
| "reward": -0.3640046715736389, |
| "reward_std": 0.5706602334976196, |
| "rewards/rollout_reward_func/mean": -0.3640046715736389, |
| "rewards/rollout_reward_func/std": 0.5570085644721985, |
| "sampling/importance_sampling_ratio/max": 2.0640242099761963, |
| "sampling/importance_sampling_ratio/mean": 1.001070499420166, |
| "sampling/importance_sampling_ratio/min": 0.4299520254135132, |
| "sampling/sampling_logp_difference/max": 0.8440816402435303, |
| "sampling/sampling_logp_difference/mean": 0.008871052414178848, |
| "step": 15, |
| "step_time": 95.68331051399582 |
| }, |
| { |
| "clip_ratio/high_max": 0.009986817487515509, |
| "clip_ratio/high_mean": 0.005344532371964306, |
| "clip_ratio/low_mean": 0.005893140041735023, |
| "clip_ratio/low_min": 0.0014044943964108825, |
| "clip_ratio/region_mean": 0.01123767253011465, |
| "entropy": 0.0662008763756603, |
| "epoch": 0.00016, |
| "grad_norm": 1.10405695438385, |
| "kl": 0.022447960553108715, |
| "learning_rate": 4.2857142857142856e-05, |
| "loss": 0.045, |
| "step": 16, |
| "step_time": 27.621052484992106 |
| }, |
| { |
| "clip_ratio/high_max": 0.006080338731408119, |
| "clip_ratio/high_mean": 0.0030401693657040596, |
| "clip_ratio/low_mean": 0.003163026151014492, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.0062031955167185515, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9357.0, |
| "completions/max_terminated_length": 9357.0, |
| "completions/mean_length": 7752.09375, |
| "completions/mean_terminated_length": 7752.09375, |
| "completions/min_length": 2165.0, |
| "completions/min_terminated_length": 2165.0, |
| "entropy": 0.07411375921219587, |
| "epoch": 0.00017, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.4593960046768188, |
| "kl": 0.05461276904679835, |
| "learning_rate": 4.5714285714285716e-05, |
| "loss": 0.0396, |
| "num_tokens": 2497538.0, |
| "reward": -0.2799840569496155, |
| "reward_std": 0.5249596834182739, |
| "rewards/rollout_reward_func/mean": -0.2799840569496155, |
| "rewards/rollout_reward_func/std": 0.5806505680084229, |
| "sampling/importance_sampling_ratio/max": 1.9463616609573364, |
| "sampling/importance_sampling_ratio/mean": 0.9976865649223328, |
| "sampling/importance_sampling_ratio/min": 0.06184462085366249, |
| "sampling/sampling_logp_difference/max": 2.783130168914795, |
| "sampling/sampling_logp_difference/mean": 0.014597892761230469, |
| "step": 17, |
| "step_time": 93.46702274899872 |
| }, |
| { |
| "clip_ratio/high_max": 0.01004306401591748, |
| "clip_ratio/high_mean": 0.00502153200795874, |
| "clip_ratio/low_mean": 0.009563877363689244, |
| "clip_ratio/low_min": 0.001420638000126928, |
| "clip_ratio/region_mean": 0.014585409400751814, |
| "entropy": 0.07483723293989897, |
| "epoch": 0.00018, |
| "grad_norm": 1.2538055181503296, |
| "kl": 0.17985437362222, |
| "learning_rate": 4.8571428571428576e-05, |
| "loss": 0.036, |
| "step": 18, |
| "step_time": 27.61892309099494 |
| }, |
| { |
| "clip_ratio/high_max": 0.007797640864737332, |
| "clip_ratio/high_mean": 0.004960351681802422, |
| "clip_ratio/low_mean": 0.003771967807551846, |
| "clip_ratio/low_min": 0.0013297871919348836, |
| "clip_ratio/region_mean": 0.008732319489354268, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9323.0, |
| "completions/max_terminated_length": 9323.0, |
| "completions/mean_length": 8057.5625, |
| "completions/mean_terminated_length": 8057.5625, |
| "completions/min_length": 4131.0, |
| "completions/min_terminated_length": 4131.0, |
| "entropy": 0.09318487590644509, |
| "epoch": 0.00019, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.8037179708480835, |
| "kl": 0.08203080226667225, |
| "learning_rate": 5.142857142857143e-05, |
| "loss": -0.1161, |
| "num_tokens": 2782910.0, |
| "reward": -0.5081856846809387, |
| "reward_std": 0.34735041856765747, |
| "rewards/rollout_reward_func/mean": -0.5081856846809387, |
| "rewards/rollout_reward_func/std": 0.34861209988594055, |
| "sampling/importance_sampling_ratio/max": 2.6022117137908936, |
| "sampling/importance_sampling_ratio/mean": 1.0005253553390503, |
| "sampling/importance_sampling_ratio/min": 0.39166536927223206, |
| "sampling/sampling_logp_difference/max": 0.9563617706298828, |
| "sampling/sampling_logp_difference/mean": 0.016585635021328926, |
| "step": 19, |
| "step_time": 96.19064523599809 |
| }, |
| { |
| "clip_ratio/high_max": 0.009210843476466835, |
| "clip_ratio/high_mean": 0.0056830079411156476, |
| "clip_ratio/low_mean": 0.01183583348756656, |
| "clip_ratio/low_min": 0.005575570685323328, |
| "clip_ratio/region_mean": 0.01751884145778604, |
| "entropy": 0.08642621210310608, |
| "epoch": 0.0002, |
| "grad_norm": 1.6954180002212524, |
| "kl": 0.1374246042687446, |
| "learning_rate": 5.428571428571428e-05, |
| "loss": -0.1201, |
| "step": 20, |
| "step_time": 27.602408401988214 |
| }, |
| { |
| "clip_ratio/high_max": 0.005014301335904747, |
| "clip_ratio/high_mean": 0.002883656619815156, |
| "clip_ratio/low_mean": 0.0028673816996160895, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.005751038319431245, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9403.0, |
| "completions/max_terminated_length": 9403.0, |
| "completions/mean_length": 7930.5625, |
| "completions/mean_terminated_length": 7930.5625, |
| "completions/min_length": 634.0, |
| "completions/min_terminated_length": 634.0, |
| "entropy": 0.07444759807549417, |
| "epoch": 0.00021, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.1838347911834717, |
| "kl": 0.09432762704091147, |
| "learning_rate": 5.714285714285714e-05, |
| "loss": -0.159, |
| "num_tokens": 3064148.0, |
| "reward": -0.3027535676956177, |
| "reward_std": 0.49178069829940796, |
| "rewards/rollout_reward_func/mean": -0.3027535676956177, |
| "rewards/rollout_reward_func/std": 0.5297390818595886, |
| "sampling/importance_sampling_ratio/max": 2.3851823806762695, |
| "sampling/importance_sampling_ratio/mean": 1.0000920295715332, |
| "sampling/importance_sampling_ratio/min": 0.3727108836174011, |
| "sampling/sampling_logp_difference/max": 0.9869523048400879, |
| "sampling/sampling_logp_difference/mean": 0.011310078203678131, |
| "step": 21, |
| "step_time": 91.98894712300535 |
| }, |
| { |
| "clip_ratio/high_max": 0.011702421237714589, |
| "clip_ratio/high_mean": 0.0065865046926774085, |
| "clip_ratio/low_mean": 0.011994066560873762, |
| "clip_ratio/low_min": 0.0024457787512801588, |
| "clip_ratio/region_mean": 0.01858057125355117, |
| "entropy": 0.07004613918252289, |
| "epoch": 0.00022, |
| "grad_norm": 1.0932140350341797, |
| "kl": 0.06876913830637932, |
| "learning_rate": 6e-05, |
| "loss": -0.168, |
| "step": 22, |
| "step_time": 27.827071762007108 |
| }, |
| { |
| "clip_ratio/high_max": 0.007137945271097124, |
| "clip_ratio/high_mean": 0.003936619672458619, |
| "clip_ratio/low_mean": 0.0028658679220825434, |
| "clip_ratio/low_min": 0.0007267441833391786, |
| "clip_ratio/region_mean": 0.006802487594541162, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9274.0, |
| "completions/max_terminated_length": 9274.0, |
| "completions/mean_length": 8095.0625, |
| "completions/mean_terminated_length": 8095.0625, |
| "completions/min_length": 1533.0, |
| "completions/min_terminated_length": 1533.0, |
| "entropy": 0.0666239457204938, |
| "epoch": 0.00023, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2763183116912842, |
| "kl": 0.05845395056530833, |
| "learning_rate": 6.285714285714286e-05, |
| "loss": 0.3004, |
| "num_tokens": 3350379.0, |
| "reward": -0.3713611960411072, |
| "reward_std": 0.44723719358444214, |
| "rewards/rollout_reward_func/mean": -0.3713611960411072, |
| "rewards/rollout_reward_func/std": 0.5041891932487488, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.0009047985076904, |
| "sampling/importance_sampling_ratio/min": 0.4158317744731903, |
| "sampling/sampling_logp_difference/max": 1.114884614944458, |
| "sampling/sampling_logp_difference/mean": 0.01109105534851551, |
| "step": 23, |
| "step_time": 93.21978440000748 |
| }, |
| { |
| "clip_ratio/high_max": 0.014727697416674346, |
| "clip_ratio/high_mean": 0.00882161219487898, |
| "clip_ratio/low_mean": 0.01083813124569133, |
| "clip_ratio/low_min": 0.003661759605165571, |
| "clip_ratio/region_mean": 0.01965974346967414, |
| "entropy": 0.06382836727425456, |
| "epoch": 0.00024, |
| "grad_norm": 1.133302092552185, |
| "kl": 0.07930020266212523, |
| "learning_rate": 6.571428571428571e-05, |
| "loss": 0.2957, |
| "step": 24, |
| "step_time": 28.035423179004283 |
| }, |
| { |
| "clip_ratio/high_max": 0.004302181303501129, |
| "clip_ratio/high_mean": 0.0021510906517505646, |
| "clip_ratio/low_mean": 0.0007800387684255838, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0029311294783838093, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8580.0, |
| "completions/max_terminated_length": 8580.0, |
| "completions/mean_length": 6758.75, |
| "completions/mean_terminated_length": 6758.75, |
| "completions/min_length": 272.0, |
| "completions/min_terminated_length": 272.0, |
| "entropy": 0.04931171424686909, |
| "epoch": 0.00025, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.291664719581604, |
| "kl": 0.07758830557577312, |
| "learning_rate": 6.857142857142858e-05, |
| "loss": -0.2042, |
| "num_tokens": 3593214.0, |
| "reward": -0.5959107875823975, |
| "reward_std": 0.35876262187957764, |
| "rewards/rollout_reward_func/mean": -0.5959107875823975, |
| "rewards/rollout_reward_func/std": 0.39302268624305725, |
| "sampling/importance_sampling_ratio/max": 2.010491371154785, |
| "sampling/importance_sampling_ratio/mean": 1.0006413459777832, |
| "sampling/importance_sampling_ratio/min": 0.42770469188690186, |
| "sampling/sampling_logp_difference/max": 0.8493223190307617, |
| "sampling/sampling_logp_difference/mean": 0.008722992613911629, |
| "step": 25, |
| "step_time": 86.72631602899492 |
| }, |
| { |
| "clip_ratio/high_max": 0.010786514729261398, |
| "clip_ratio/high_mean": 0.005393257364630699, |
| "clip_ratio/low_mean": 0.011662953183986247, |
| "clip_ratio/low_min": 0.0019696029485203326, |
| "clip_ratio/region_mean": 0.017056210490409285, |
| "entropy": 0.05728732608258724, |
| "epoch": 0.00026, |
| "grad_norm": 2.188659906387329, |
| "kl": 0.08259193820413202, |
| "learning_rate": 7.142857142857143e-05, |
| "loss": -0.2036, |
| "step": 26, |
| "step_time": 26.119606291009404 |
| }, |
| { |
| "clip_ratio/high_max": 0.0061968869413249195, |
| "clip_ratio/high_mean": 0.0030984434706624597, |
| "clip_ratio/low_mean": 0.0010736910917330533, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0041721345914993435, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8940.0, |
| "completions/max_terminated_length": 8940.0, |
| "completions/mean_length": 7771.3125, |
| "completions/mean_terminated_length": 7771.3125, |
| "completions/min_length": 1515.0, |
| "completions/min_terminated_length": 1515.0, |
| "entropy": 0.0601597810164094, |
| "epoch": 0.00027, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.141928791999817, |
| "kl": 0.05808806075947359, |
| "learning_rate": 7.428571428571429e-05, |
| "loss": 0.1398, |
| "num_tokens": 3868467.0, |
| "reward": -0.36368632316589355, |
| "reward_std": 0.5357357263565063, |
| "rewards/rollout_reward_func/mean": -0.36368632316589355, |
| "rewards/rollout_reward_func/std": 0.5756723880767822, |
| "sampling/importance_sampling_ratio/max": 1.8575561046600342, |
| "sampling/importance_sampling_ratio/mean": 1.0000970363616943, |
| "sampling/importance_sampling_ratio/min": 0.3134206235408783, |
| "sampling/sampling_logp_difference/max": 1.1602091789245605, |
| "sampling/sampling_logp_difference/mean": 0.009103494696319103, |
| "step": 27, |
| "step_time": 92.96613138498651 |
| }, |
| { |
| "clip_ratio/high_max": 0.0164202400483191, |
| "clip_ratio/high_mean": 0.00821012002415955, |
| "clip_ratio/low_mean": 0.013974322064314038, |
| "clip_ratio/low_min": 0.0007183908019214869, |
| "clip_ratio/region_mean": 0.022184442059369758, |
| "entropy": 0.0649251660797745, |
| "epoch": 0.00028, |
| "grad_norm": 0.8147655725479126, |
| "kl": 0.10712331312242895, |
| "learning_rate": 7.714285714285715e-05, |
| "loss": 0.1369, |
| "step": 28, |
| "step_time": 27.32312696599911 |
| }, |
| { |
| "clip_ratio/high_max": 0.002155932132154703, |
| "clip_ratio/high_mean": 0.0010779660660773516, |
| "clip_ratio/low_mean": 0.0007102272938936949, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0017881933599710464, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9165.0, |
| "completions/max_terminated_length": 9165.0, |
| "completions/mean_length": 7336.8125, |
| "completions/mean_terminated_length": 7336.8125, |
| "completions/min_length": 1667.0, |
| "completions/min_terminated_length": 1667.0, |
| "entropy": 0.06351292878389359, |
| "epoch": 0.00029, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.7936819195747375, |
| "kl": 0.04702634894056246, |
| "learning_rate": 8e-05, |
| "loss": -0.2535, |
| "num_tokens": 4129721.0, |
| "reward": -0.4376564919948578, |
| "reward_std": 0.39146924018859863, |
| "rewards/rollout_reward_func/mean": -0.4376564919948578, |
| "rewards/rollout_reward_func/std": 0.4162859320640564, |
| "sampling/importance_sampling_ratio/max": 1.9884843826293945, |
| "sampling/importance_sampling_ratio/mean": 1.0006194114685059, |
| "sampling/importance_sampling_ratio/min": 0.5153306126594543, |
| "sampling/sampling_logp_difference/max": 0.6873726844787598, |
| "sampling/sampling_logp_difference/mean": 0.00740803312510252, |
| "step": 29, |
| "step_time": 91.82200075300352 |
| }, |
| { |
| "clip_ratio/high_max": 0.005039906303863972, |
| "clip_ratio/high_mean": 0.00323018038761802, |
| "clip_ratio/low_mean": 0.010294274339685217, |
| "clip_ratio/low_min": 0.004024621332064271, |
| "clip_ratio/region_mean": 0.013524454698199406, |
| "entropy": 0.06765187811106443, |
| "epoch": 0.0003, |
| "grad_norm": 0.7302473783493042, |
| "kl": 0.050249568943399936, |
| "learning_rate": 8.285714285714287e-05, |
| "loss": -0.2607, |
| "step": 30, |
| "step_time": 28.11271668801055 |
| }, |
| { |
| "clip_ratio/high_max": 0.00493967067450285, |
| "clip_ratio/high_mean": 0.002469835337251425, |
| "clip_ratio/low_mean": 0.0007143090479075909, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0031841443851590157, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9146.0, |
| "completions/max_terminated_length": 9146.0, |
| "completions/mean_length": 7793.5625, |
| "completions/mean_terminated_length": 7793.5625, |
| "completions/min_length": 263.0, |
| "completions/min_terminated_length": 263.0, |
| "entropy": 0.08614894514903426, |
| "epoch": 0.00031, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0549495220184326, |
| "kl": 0.06513052270747721, |
| "learning_rate": 8.571428571428571e-05, |
| "loss": -0.0626, |
| "num_tokens": 4405861.0, |
| "reward": -0.37940478324890137, |
| "reward_std": 0.45117825269699097, |
| "rewards/rollout_reward_func/mean": -0.37940478324890137, |
| "rewards/rollout_reward_func/std": 0.48283621668815613, |
| "sampling/importance_sampling_ratio/max": 1.4535597562789917, |
| "sampling/importance_sampling_ratio/mean": 0.9976645112037659, |
| "sampling/importance_sampling_ratio/min": 0.5075654983520508, |
| "sampling/sampling_logp_difference/max": 0.6781294345855713, |
| "sampling/sampling_logp_difference/mean": 0.009683198295533657, |
| "step": 31, |
| "step_time": 92.44875282800785 |
| }, |
| { |
| "clip_ratio/high_max": 0.011355375638231635, |
| "clip_ratio/high_mean": 0.006735137314535677, |
| "clip_ratio/low_mean": 0.012324601178988814, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.019059738871874288, |
| "entropy": 0.08400337002240121, |
| "epoch": 0.00032, |
| "grad_norm": 0.6007691621780396, |
| "kl": 0.08906090061645955, |
| "learning_rate": 8.857142857142857e-05, |
| "loss": -0.0698, |
| "step": 32, |
| "step_time": 28.252189476006606 |
| }, |
| { |
| "clip_ratio/high_max": 0.007888395455665886, |
| "clip_ratio/high_mean": 0.003944197727832943, |
| "clip_ratio/low_mean": 0.004384168569231406, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.00832836632616818, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9394.0, |
| "completions/max_terminated_length": 9394.0, |
| "completions/mean_length": 7661.125, |
| "completions/mean_terminated_length": 7661.125, |
| "completions/min_length": 1012.0, |
| "completions/min_terminated_length": 1012.0, |
| "entropy": 0.08313656039535999, |
| "epoch": 0.00033, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.035186767578125, |
| "kl": 0.09069790283683687, |
| "learning_rate": 9.142857142857143e-05, |
| "loss": 0.0166, |
| "num_tokens": 4677397.0, |
| "reward": -0.3240296244621277, |
| "reward_std": 0.4347441494464874, |
| "rewards/rollout_reward_func/mean": -0.3240296244621277, |
| "rewards/rollout_reward_func/std": 0.4854819178581238, |
| "sampling/importance_sampling_ratio/max": 1.7231584787368774, |
| "sampling/importance_sampling_ratio/mean": 0.9987027645111084, |
| "sampling/importance_sampling_ratio/min": 0.06808681786060333, |
| "sampling/sampling_logp_difference/max": 2.686971664428711, |
| "sampling/sampling_logp_difference/mean": 0.010714043863117695, |
| "step": 33, |
| "step_time": 93.2897496730111 |
| }, |
| { |
| "clip_ratio/high_max": 0.020736676699016243, |
| "clip_ratio/high_mean": 0.01142766050179489, |
| "clip_ratio/low_mean": 0.010279498674208298, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.02170715932152234, |
| "entropy": 0.07817789679393172, |
| "epoch": 0.00034, |
| "grad_norm": 0.6795229911804199, |
| "kl": 0.110447074577678, |
| "learning_rate": 9.428571428571429e-05, |
| "loss": 0.0053, |
| "step": 34, |
| "step_time": 28.798946735994832 |
| }, |
| { |
| "clip_ratio/high_max": 0.004271204466931522, |
| "clip_ratio/high_mean": 0.002135602233465761, |
| "clip_ratio/low_mean": 0.0010695144010242075, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0032051166344899684, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9245.0, |
| "completions/max_terminated_length": 9245.0, |
| "completions/mean_length": 8244.75, |
| "completions/mean_terminated_length": 8244.75, |
| "completions/min_length": 291.0, |
| "completions/min_terminated_length": 291.0, |
| "entropy": 0.06729014380834997, |
| "epoch": 0.00035, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.4786404371261597, |
| "kl": 0.10362116701435298, |
| "learning_rate": 9.714285714285715e-05, |
| "loss": -0.2709, |
| "num_tokens": 4968197.0, |
| "reward": -0.3916197419166565, |
| "reward_std": 0.35996532440185547, |
| "rewards/rollout_reward_func/mean": -0.3916197419166565, |
| "rewards/rollout_reward_func/std": 0.3666594326496124, |
| "sampling/importance_sampling_ratio/max": 1.5868641138076782, |
| "sampling/importance_sampling_ratio/mean": 1.0002307891845703, |
| "sampling/importance_sampling_ratio/min": 0.6188454031944275, |
| "sampling/sampling_logp_difference/max": 0.4798997640609741, |
| "sampling/sampling_logp_difference/mean": 0.007114737294614315, |
| "step": 35, |
| "step_time": 92.1033384639959 |
| }, |
| { |
| "clip_ratio/high_max": 0.017084260645788163, |
| "clip_ratio/high_mean": 0.009252357733203098, |
| "clip_ratio/low_mean": 0.012271191226318479, |
| "clip_ratio/low_min": 0.0014044943964108825, |
| "clip_ratio/region_mean": 0.021523548872210085, |
| "entropy": 0.06766591034829617, |
| "epoch": 0.00036, |
| "grad_norm": 0.7554082870483398, |
| "kl": 0.11198568157851696, |
| "learning_rate": 0.0001, |
| "loss": -0.2783, |
| "step": 36, |
| "step_time": 28.797730328002217 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014046717551536858, |
| "clip_ratio/high_mean": 0.0007023358775768429, |
| "clip_ratio/low_mean": 0.0, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0007023358775768429, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9282.0, |
| "completions/max_terminated_length": 9282.0, |
| "completions/mean_length": 7701.8125, |
| "completions/mean_terminated_length": 7701.8125, |
| "completions/min_length": 1350.0, |
| "completions/min_terminated_length": 1350.0, |
| "entropy": 0.06278971070423722, |
| "epoch": 0.00037, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.0583816766738892, |
| "kl": 0.12011543358676136, |
| "learning_rate": 9.918053502751771e-05, |
| "loss": -0.0297, |
| "num_tokens": 5241387.0, |
| "reward": -0.304426908493042, |
| "reward_std": 0.4464525580406189, |
| "rewards/rollout_reward_func/mean": -0.304426908493042, |
| "rewards/rollout_reward_func/std": 0.4538337290287018, |
| "sampling/importance_sampling_ratio/max": 1.6482372283935547, |
| "sampling/importance_sampling_ratio/mean": 0.9995368719100952, |
| "sampling/importance_sampling_ratio/min": 0.5574944615364075, |
| "sampling/sampling_logp_difference/max": 0.5843026638031006, |
| "sampling/sampling_logp_difference/mean": 0.007077127229422331, |
| "step": 37, |
| "step_time": 91.12368497201533 |
| }, |
| { |
| "clip_ratio/high_max": 0.010138559562619776, |
| "clip_ratio/high_mean": 0.006660482700681314, |
| "clip_ratio/low_mean": 0.01362931149196811, |
| "clip_ratio/low_min": 0.003907083009835333, |
| "clip_ratio/region_mean": 0.020289794192649424, |
| "entropy": 0.05679044884163886, |
| "epoch": 0.00038, |
| "grad_norm": 0.618403971195221, |
| "kl": 0.118131990195252, |
| "learning_rate": 9.675795466159755e-05, |
| "loss": -0.0362, |
| "step": 38, |
| "step_time": 28.32057495099798 |
| }, |
| { |
| "clip_ratio/high_max": 0.005428613570984453, |
| "clip_ratio/high_mean": 0.0027143067854922265, |
| "clip_ratio/low_mean": 0.0010573608451522887, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003771667630644515, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9203.0, |
| "completions/max_terminated_length": 9203.0, |
| "completions/mean_length": 7459.09375, |
| "completions/mean_terminated_length": 7459.09375, |
| "completions/min_length": 647.0, |
| "completions/min_terminated_length": 647.0, |
| "entropy": 0.05742924939841032, |
| "epoch": 0.00039, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.1807667016983032, |
| "kl": 0.11824507999699563, |
| "learning_rate": 9.283813728906054e-05, |
| "loss": -0.0685, |
| "num_tokens": 5507267.0, |
| "reward": -0.1265014261007309, |
| "reward_std": 0.6104403734207153, |
| "rewards/rollout_reward_func/mean": -0.1265014261007309, |
| "rewards/rollout_reward_func/std": 0.6330578327178955, |
| "sampling/importance_sampling_ratio/max": 1.6747703552246094, |
| "sampling/importance_sampling_ratio/mean": 0.9992620944976807, |
| "sampling/importance_sampling_ratio/min": 0.39614981412887573, |
| "sampling/sampling_logp_difference/max": 0.9259628653526306, |
| "sampling/sampling_logp_difference/mean": 0.006985975429415703, |
| "step": 39, |
| "step_time": 91.56473480099157 |
| }, |
| { |
| "clip_ratio/high_max": 0.017474214837420732, |
| "clip_ratio/high_mean": 0.008737107418710366, |
| "clip_ratio/low_mean": 0.01038203525240533, |
| "clip_ratio/low_min": 0.0021973324473947287, |
| "clip_ratio/region_mean": 0.01911914284573868, |
| "entropy": 0.05913685623090714, |
| "epoch": 0.0004, |
| "grad_norm": 11257.0869140625, |
| "kl": 15902.953878552886, |
| "learning_rate": 8.759239773845718e-05, |
| "loss": 59.5637, |
| "step": 40, |
| "step_time": 28.143666776995815 |
| }, |
| { |
| "clip_ratio/high_max": 0.0014367816038429737, |
| "clip_ratio/high_mean": 0.00109489681199193, |
| "clip_ratio/low_mean": 0.002375774463871494, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003470671275863424, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9299.0, |
| "completions/max_terminated_length": 9299.0, |
| "completions/mean_length": 7399.59375, |
| "completions/mean_terminated_length": 7399.59375, |
| "completions/min_length": 311.0, |
| "completions/min_terminated_length": 311.0, |
| "entropy": 0.0707793568726629, |
| "epoch": 0.00041, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.779978334903717, |
| "kl": 0.1064646802842617, |
| "learning_rate": 8.125000000000001e-05, |
| "loss": -0.006, |
| "num_tokens": 5770826.0, |
| "reward": -0.10253402590751648, |
| "reward_std": 0.587022602558136, |
| "rewards/rollout_reward_func/mean": -0.10253402590751648, |
| "rewards/rollout_reward_func/std": 0.58503657579422, |
| "sampling/importance_sampling_ratio/max": 1.7891288995742798, |
| "sampling/importance_sampling_ratio/mean": 1.000353455543518, |
| "sampling/importance_sampling_ratio/min": 0.33879733085632324, |
| "sampling/sampling_logp_difference/max": 1.082353115081787, |
| "sampling/sampling_logp_difference/mean": 0.008459154516458511, |
| "step": 41, |
| "step_time": 89.9564655849972 |
| }, |
| { |
| "clip_ratio/high_max": 0.013633344613481313, |
| "clip_ratio/high_mean": 0.0071758677077014, |
| "clip_ratio/low_mean": 0.013401742442511022, |
| "clip_ratio/low_min": 0.0028420357266440988, |
| "clip_ratio/region_mean": 0.020577610324835405, |
| "entropy": 0.07488202536478639, |
| "epoch": 0.00042, |
| "grad_norm": 0.6732333302497864, |
| "kl": 0.11995455622673035, |
| "learning_rate": 7.408813728906053e-05, |
| "loss": -0.0139, |
| "step": 42, |
| "step_time": 27.865521725998406 |
| }, |
| { |
| "clip_ratio/high_max": 0.001420638000126928, |
| "clip_ratio/high_mean": 0.000710319000063464, |
| "clip_ratio/low_mean": 0.002360046637477353, |
| "clip_ratio/low_min": 0.0007022471982054412, |
| "clip_ratio/region_mean": 0.0030703656375408173, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 8920.0, |
| "completions/max_terminated_length": 8920.0, |
| "completions/mean_length": 7293.5, |
| "completions/mean_terminated_length": 7293.5, |
| "completions/min_length": 876.0, |
| "completions/min_terminated_length": 876.0, |
| "entropy": 0.06767579750157893, |
| "epoch": 0.00043, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 0.9846507906913757, |
| "kl": 0.11756309028714895, |
| "learning_rate": 6.6419817372537e-05, |
| "loss": -0.105, |
| "num_tokens": 6030904.0, |
| "reward": -0.17516297101974487, |
| "reward_std": 0.6336148977279663, |
| "rewards/rollout_reward_func/mean": -0.17516297101974487, |
| "rewards/rollout_reward_func/std": 0.6060534715652466, |
| "sampling/importance_sampling_ratio/max": 1.5995110273361206, |
| "sampling/importance_sampling_ratio/mean": 1.0016493797302246, |
| "sampling/importance_sampling_ratio/min": 0.6211615800857544, |
| "sampling/sampling_logp_difference/max": 0.4761640429496765, |
| "sampling/sampling_logp_difference/mean": 0.00795302540063858, |
| "step": 43, |
| "step_time": 88.68364224100515 |
| }, |
| { |
| "clip_ratio/high_max": 0.011437574110459536, |
| "clip_ratio/high_mean": 0.006488003156846389, |
| "clip_ratio/low_mean": 0.011332564434269443, |
| "clip_ratio/low_min": 0.002825683041010052, |
| "clip_ratio/region_mean": 0.01782056744559668, |
| "entropy": 0.06705769780091941, |
| "epoch": 0.00044, |
| "grad_norm": 0.5778977274894714, |
| "kl": 0.13579032279085368, |
| "learning_rate": 5.858018262746301e-05, |
| "loss": -0.1148, |
| "step": 44, |
| "step_time": 26.98718312199344 |
| }, |
| { |
| "clip_ratio/high_max": 0.0022492663701996207, |
| "clip_ratio/high_mean": 0.0011246331850998104, |
| "clip_ratio/low_mean": 0.0028377448907122016, |
| "clip_ratio/low_min": 0.0006944444612599909, |
| "clip_ratio/region_mean": 0.003962378075812012, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9242.0, |
| "completions/max_terminated_length": 9242.0, |
| "completions/mean_length": 8146.5, |
| "completions/mean_terminated_length": 8146.5, |
| "completions/min_length": 1426.0, |
| "completions/min_terminated_length": 1426.0, |
| "entropy": 0.08103846106678247, |
| "epoch": 0.00045, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.409741759300232, |
| "kl": 0.14359550992958248, |
| "learning_rate": 5.0911862710939485e-05, |
| "loss": -0.034, |
| "num_tokens": 6318667.0, |
| "reward": -0.12613427639007568, |
| "reward_std": 0.5343547463417053, |
| "rewards/rollout_reward_func/mean": -0.12613427639007568, |
| "rewards/rollout_reward_func/std": 0.5416647791862488, |
| "sampling/importance_sampling_ratio/max": 3.0, |
| "sampling/importance_sampling_ratio/mean": 1.000476598739624, |
| "sampling/importance_sampling_ratio/min": 0.09892720729112625, |
| "sampling/sampling_logp_difference/max": 2.313370943069458, |
| "sampling/sampling_logp_difference/mean": 0.012712076306343079, |
| "step": 45, |
| "step_time": 93.79370664700036 |
| }, |
| { |
| "clip_ratio/high_max": 0.011134565633255988, |
| "clip_ratio/high_mean": 0.006269618694204837, |
| "clip_ratio/low_mean": 0.009181194531265646, |
| "clip_ratio/low_min": 0.0028904599603265524, |
| "clip_ratio/region_mean": 0.015450813429197297, |
| "entropy": 0.08389568043639883, |
| "epoch": 0.00046, |
| "grad_norm": 1.2059129476547241, |
| "kl": 0.24020475335419178, |
| "learning_rate": 4.375000000000001e-05, |
| "loss": -0.0432, |
| "step": 46, |
| "step_time": 28.168583785994997 |
| }, |
| { |
| "clip_ratio/high_max": 0.005004245089367032, |
| "clip_ratio/high_mean": 0.002502122544683516, |
| "clip_ratio/low_mean": 0.002046188630629331, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.0045483111462090164, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9056.0, |
| "completions/max_terminated_length": 9056.0, |
| "completions/mean_length": 7947.84375, |
| "completions/mean_terminated_length": 7947.84375, |
| "completions/min_length": 4376.0, |
| "completions/min_terminated_length": 4376.0, |
| "entropy": 0.07940921117551625, |
| "epoch": 0.00047, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.2504568099975586, |
| "kl": 0.16516069741919637, |
| "learning_rate": 3.740760226154283e-05, |
| "loss": -0.0286, |
| "num_tokens": 6600118.0, |
| "reward": -0.12103764712810516, |
| "reward_std": 0.5211879014968872, |
| "rewards/rollout_reward_func/mean": -0.12103764712810516, |
| "rewards/rollout_reward_func/std": 0.5456745028495789, |
| "sampling/importance_sampling_ratio/max": 1.9942717552185059, |
| "sampling/importance_sampling_ratio/mean": 0.9991114139556885, |
| "sampling/importance_sampling_ratio/min": 0.11994559317827225, |
| "sampling/sampling_logp_difference/max": 2.1207170486450195, |
| "sampling/sampling_logp_difference/mean": 0.010312505066394806, |
| "step": 47, |
| "step_time": 94.76915054599158 |
| }, |
| { |
| "clip_ratio/high_max": 0.00876080454327166, |
| "clip_ratio/high_mean": 0.00438040227163583, |
| "clip_ratio/low_mean": 0.002355890115723014, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.006736292387358844, |
| "entropy": 0.08134529506787658, |
| "epoch": 0.00048, |
| "grad_norm": 0.805917501449585, |
| "kl": 0.16713016014546156, |
| "learning_rate": 3.2161862710939476e-05, |
| "loss": -0.0331, |
| "step": 48, |
| "step_time": 27.11353817199779 |
| }, |
| { |
| "clip_ratio/high_max": 0.004328028007876128, |
| "clip_ratio/high_mean": 0.002164014003938064, |
| "clip_ratio/low_mean": 0.0010534594766795635, |
| "clip_ratio/low_min": 0.0, |
| "clip_ratio/region_mean": 0.003217473509721458, |
| "completions/clipped_ratio": 0.0, |
| "completions/max_length": 9558.0, |
| "completions/max_terminated_length": 9558.0, |
| "completions/mean_length": 8071.4375, |
| "completions/mean_terminated_length": 8071.4375, |
| "completions/min_length": 3757.0, |
| "completions/min_terminated_length": 3757.0, |
| "entropy": 0.09437589161098003, |
| "epoch": 0.00049, |
| "frac_reward_zero_std": 0.0, |
| "grad_norm": 1.223923921585083, |
| "kl": 0.17360988864675164, |
| "learning_rate": 2.8242045338402468e-05, |
| "loss": 0.0868, |
| "num_tokens": 6885078.0, |
| "reward": -0.17595598101615906, |
| "reward_std": 0.574884295463562, |
| "rewards/rollout_reward_func/mean": -0.17595598101615906, |
| "rewards/rollout_reward_func/std": 0.617475152015686, |
| "sampling/importance_sampling_ratio/max": 2.3295459747314453, |
| "sampling/importance_sampling_ratio/mean": 0.9994887709617615, |
| "sampling/importance_sampling_ratio/min": 0.21804022789001465, |
| "sampling/sampling_logp_difference/max": 1.5230756998062134, |
| "sampling/sampling_logp_difference/mean": 0.012081097811460495, |
| "step": 49, |
| "step_time": 92.44432610799413 |
| }, |
| { |
| "clip_ratio/high_max": 0.009452320169657469, |
| "clip_ratio/high_mean": 0.005073382286354899, |
| "clip_ratio/low_mean": 0.00776787509676069, |
| "clip_ratio/low_min": 0.002785580581985414, |
| "clip_ratio/region_mean": 0.012841257266700268, |
| "entropy": 0.09734066273085773, |
| "epoch": 0.0005, |
| "grad_norm": 1.335457682609558, |
| "kl": 0.17287601297721267, |
| "learning_rate": 2.5819464972482288e-05, |
| "loss": 0.0771, |
| "step": 50, |
| "step_time": 28.072686168001383 |
| } |
| ], |
| "logging_steps": 1.0, |
| "max_steps": 50, |
| "num_input_tokens_seen": 6885078, |
| "num_train_epochs": 1, |
| "save_steps": 500, |
| "stateful_callbacks": { |
| "TrainerControl": { |
| "args": { |
| "should_epoch_stop": false, |
| "should_evaluate": false, |
| "should_log": false, |
| "should_save": true, |
| "should_training_stop": true |
| }, |
| "attributes": {} |
| } |
| }, |
| "total_flos": 0.0, |
| "train_batch_size": 1, |
| "trial_name": null, |
| "trial_params": null |
| } |
|
|
